xds: Add v3 support for client bootstrap. (#3723)

xds/internal/balancer/edsbalancer/eds_test.go

@@ -25,7 +25,6 @@ import (
 	"reflect"
 	"testing"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/golang/protobuf/jsonpb"
 	wrapperspb "github.com/golang/protobuf/ptypes/wrappers"
 	"github.com/google/go-cmp/cmp"
@@ -51,7 +50,7 @@ func init() {
 		return &bootstrap.Config{
 			BalancerName: testBalancerNameFooBar,
 			Creds:        grpc.WithInsecure(),
-			NodeProto:    &corepb.Node{},
+			NodeProto:    testutils.EmptyNodeProtoV2,
 		}, nil
 	}
 }

xds/internal/balancer/edsbalancer/xds_client_wrapper_test.go

@@ -24,7 +24,6 @@ import (
 	"time"
 
 	xdspb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
 	"github.com/golang/protobuf/proto"
 	"github.com/google/go-cmp/cmp"
 	"google.golang.org/grpc"
@@ -101,7 +100,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 				return &bootstrap.Config{
 					BalancerName: fakeServer.Address,
 					Creds:        grpc.WithInsecure(),
-					NodeProto:    &corepb.Node{},
+					NodeProto:    testutils.EmptyNodeProtoV2,
 				}, nil
 			}
 			defer func() { bootstrapConfigNew = oldBootstrapConfigNew }()
@@ -138,7 +137,7 @@ func (s) TestClientWrapperWatchEDS(t *testing.T) {
 			wantReq := &xdspb.DiscoveryRequest{
 				TypeUrl:       edsType,
 				ResourceNames: []string{test.wantResourceName},
-				Node:          &corepb.Node{},
+				Node:          testutils.EmptyNodeProtoV2,
 			}
 			if !proto.Equal(edsReq.Req, wantReq) {
 				t.Fatalf("got EDS request %v, expected: %v, diff: %s", edsReq.Req, wantReq, cmp.Diff(edsReq.Req, wantReq, cmp.Comparer(proto.Equal)))

xds/internal/client/bootstrap/bootstrap.go

@@ -27,15 +27,25 @@ import (
 	"io/ioutil"
 	"os"
 
-	corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v2corepb "github.com/envoyproxy/go-control-plane/envoy/api/v2/core"
+	v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
 	"github.com/golang/protobuf/jsonpb"
+	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/xds/internal/version"
 )
 
 const (
 	// Environment variable which holds the name of the xDS bootstrap file.
-	fileEnv = "GRPC_XDS_BOOTSTRAP"
+	bootstrapFileEnv = "GRPC_XDS_BOOTSTRAP"
+	// Environment variable which controls the use of xDS v3 API.
+	v3SupportEnv = "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT"
+	// The "server_features" field in the bootstrap file contains a list of
+	// features supported by the server. A value of "xds_v3" indicates that the
+	// server supports the v3 version of the xDS transport protocol.
+	serverFeaturesV3 = "xds_v3"
+
 	// Type name for Google default credentials.
 	googleDefaultCreds              = "google_default"
 	gRPCUserAgentName               = "gRPC Go"
@@ -45,7 +55,7 @@ const (
 var gRPCVersion = fmt.Sprintf("%s %s", gRPCUserAgentName, grpc.Version)
 
 // For overriding in unit tests.
-var fileReadFunc = ioutil.ReadFile
+var bootstrapFileReadFunc = ioutil.ReadFile
 
 // Config provides the xDS client with several key bits of information that it
 // requires in its interaction with an xDS server. The Config is initialized
@@ -59,8 +69,13 @@ type Config struct {
 	// Creds contains the credentials to be used while talking to the xDS
 	// server, as a grpc.DialOption.
 	Creds grpc.DialOption
-	// NodeProto contains the node proto to be used in xDS requests.
-	NodeProto *corepb.Node
+	// TransportAPI indicates the API version of xDS transport protocol to use.
+	// This describes the xDS gRPC endpoint and version of
+	// DiscoveryRequest/Response used on the wire.
+	TransportAPI version.TransportAPI
+	// NodeProto contains the Node proto to be used in xDS requests. The actual
+	// type depends on the transport protocol version used.
+	NodeProto proto.Message
 }
 
 type channelCreds struct {
@@ -85,9 +100,10 @@ type xdsServer struct {
 //          "type": <string containing channel cred type>,
 //          "config": <JSON object containing config for the type>
 //        }
-//      ]
+//      ],
+//      "server_features": [ ... ]
 //    },
-//    "node": <JSON form of corepb.Node proto>
+//    "node": <JSON form of Node proto>
 // }
 //
 // Currently, we support exactly one type of credential, which is
@@ -101,13 +117,13 @@ type xdsServer struct {
 func NewConfig() (*Config, error) {
 	config := &Config{}
 
-	fName, ok := os.LookupEnv(fileEnv)
+	fName, ok := os.LookupEnv(bootstrapFileEnv)
 	if !ok {
-		return nil, fmt.Errorf("xds: Environment variable %v not defined", fileEnv)
+		return nil, fmt.Errorf("xds: Environment variable %v not defined", bootstrapFileEnv)
 	}
-	logger.Infof("Got bootstrap file location from %v environment variable: %v", fileEnv, fName)
+	logger.Infof("Got bootstrap file location from %v environment variable: %v", bootstrapFileEnv, fName)
 
-	data, err := fileReadFunc(fName)
+	data, err := bootstrapFileReadFunc(fName)
 	if err != nil {
 		return nil, fmt.Errorf("xds: Failed to read bootstrap file %s with error %v", fName, err)
 	}
@@ -118,11 +134,18 @@ func NewConfig() (*Config, error) {
 		return nil, fmt.Errorf("xds: Failed to parse file %s (content %v) with error: %v", fName, string(data), err)
 	}
 
+	serverSupportsV3 := false
 	m := jsonpb.Unmarshaler{AllowUnknownFields: true}
 	for k, v := range jsonData {
 		switch k {
 		case "node":
-			n := &corepb.Node{}
+			// We unconditionally convert the JSON into a v3.Node proto. The v3
+			// proto does not contain the deprecated field "build_version" from
+			// the v2 proto. We do not expect the bootstrap file to contain the
+			// "build_version" field. In any case, the unmarshal will succeed
+			// because we have set the `AllowUnknownFields` option on the
+			// unmarshaler.
+			n := &v3corepb.Node{}
 			if err := m.Unmarshal(bytes.NewReader(v), n); err != nil {
 				return nil, fmt.Errorf("xds: jsonpb.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
 			}
@@ -144,6 +167,17 @@ func NewConfig() (*Config, error) {
 					break
 				}
 			}
+		case "server_features":
+			var features []string
+			if err := json.Unmarshal(v, &features); err != nil {
+				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
+			}
+			for _, f := range features {
+				switch f {
+				case serverFeaturesV3:
+					serverSupportsV3 = true
+				}
+			}
 		}
 		// Do not fail the xDS bootstrap when an unknown field is seen. This can
 		// happen when an older version client reads a newer version bootstrap
@@ -154,20 +188,69 @@ func NewConfig() (*Config, error) {
 		return nil, fmt.Errorf("xds: Required field %q not found in bootstrap", "xds_servers.server_uri")
 	}
 
-	// If we don't find a nodeProto in the bootstrap file, we just create an
-	// empty one here. That way, callers of this function can always expect
-	// that the NodeProto field is non-nil.
-	if config.NodeProto == nil {
-		config.NodeProto = &corepb.Node{}
+	// We end up using v3 transport protocol version only if the following
+	// conditions are met:
+	// 1. Server supports v3, indicated by the presence of "xds_v3" in
+	//    server_features.
+	// 2. Environment variable "GRPC_XDS_EXPERIMENTAL_V3_SUPPORT" is set to
+	//    true.
+	// The default value of the enum type "version.TransportAPI" is v2.
+	if v3Env := os.Getenv(v3SupportEnv); v3Env == "true" {
+		if serverSupportsV3 {
+			config.TransportAPI = version.TransportV3
+		}
 	}
-	// BuildVersion is deprecated, and is replaced by user_agent_name and
-	// user_agent_version. But the management servers are still using the old
-	// field, so we will keep both set.
-	config.NodeProto.BuildVersion = gRPCVersion
-	config.NodeProto.UserAgentName = gRPCUserAgentName
-	config.NodeProto.UserAgentVersionType = &corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
-	config.NodeProto.ClientFeatures = append(config.NodeProto.ClientFeatures, clientFeatureNoOverprovisioning)
 
+	if err := config.updateNodeProto(); err != nil {
+		return nil, err
+	}
 	logger.Infof("Bootstrap config for creating xds-client: %+v", config)
 	return config, nil
 }
+
+// updateNodeProto updates the node proto read from the bootstrap file.
+//
+// Node proto in Config contains a v3.Node protobuf message corresponding to the
+// JSON contents found in the bootstrap file. This method performs some post
+// processing on it:
+// 1. If we don't find a nodeProto in the bootstrap file, we create an empty one
+// here. That way, callers of this function can always expect that the NodeProto
+// field is non-nil.
+// 2. If the transport protocol version to be used is not v3, we convert the
+// current v3.Node proto in a v2.Node proto.
+// 3. Some additional fields which are not expected to be set in the bootstrap
+// file are populated here.
+func (c *Config) updateNodeProto() error {
+	if c.TransportAPI == version.TransportV3 {
+		v3, _ := c.NodeProto.(*v3corepb.Node)
+		if v3 == nil {
+			v3 = &v3corepb.Node{}
+		}
+		v3.UserAgentName = gRPCUserAgentName
+		v3.UserAgentVersionType = &v3corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
+		v3.ClientFeatures = append(v3.ClientFeatures, clientFeatureNoOverprovisioning)
+		c.NodeProto = v3
+		return nil
+	}
+
+	v2 := &v2corepb.Node{}
+	if c.NodeProto != nil {
+		v3, err := proto.Marshal(c.NodeProto)
+		if err != nil {
+			return fmt.Errorf("xds: proto.Marshal(%v): %v", c.NodeProto, err)
+		}
+		if err := proto.Unmarshal(v3, v2); err != nil {
+			return fmt.Errorf("xds: proto.Unmarshal(%v): %v", v3, err)
+		}
+	}
+	c.NodeProto = v2
+
+	// BuildVersion is deprecated, and is replaced by user_agent_name and
+	// user_agent_version. But the management servers are still using the old
+	// field, so we will keep both set.
+	v2.BuildVersion = gRPCVersion
+	v2.UserAgentName = gRPCUserAgentName
+	v2.UserAgentVersionType = &v2corepb.Node_UserAgentVersion{UserAgentVersion: grpc.Version}
+	v2.ClientFeatures = append(v2.ClientFeatures, clientFeatureNoOverprovisioning)
+	return nil
+}