Skip to content

Commit

Permalink
xds: Use management server from go-control-plane. (#4039)
Browse files Browse the repository at this point in the history
  • Loading branch information
easwars committed Nov 18, 2020
1 parent c5cf514 commit 230166b
Show file tree
Hide file tree
Showing 4 changed files with 326 additions and 92 deletions.
5 changes: 5 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ github.com/client9/misspell v0.3.4 h1:ta993UF76GwbvJcIo3Y68y/M3WxlpEHPWIGDkJYwzJ
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354 h1:9kRtNpqLHbZVO/NNxhHp2ymxFxsHOe3x2efJGn//Tas=
github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
Expand Down Expand Up @@ -36,9 +37,11 @@ github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y=
github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
Expand Down Expand Up @@ -87,7 +90,9 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2
google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
148 changes: 56 additions & 92 deletions xds/internal/test/xds_server_integration_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// +build !386

/*
*
* Copyright 2020 gRPC authors.
Expand All @@ -22,27 +24,25 @@ package xds_test
import (
"context"
"fmt"
"io/ioutil"
"os"
"path"
"testing"
"time"

"github.com/google/uuid"
"google.golang.org/protobuf/proto"
"google.golang.org/protobuf/types/known/anypb"

v2discoverypb "github.com/envoyproxy/go-control-plane/envoy/api/v2"
v3corepb "github.com/envoyproxy/go-control-plane/envoy/config/core/v3"
v3listenerpb "github.com/envoyproxy/go-control-plane/envoy/config/listener/v3"
v3httppb "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/http_connection_manager/v3"

"google.golang.org/grpc"
"google.golang.org/grpc/credentials/insecure"
"google.golang.org/grpc/internal/grpctest"
testpb "google.golang.org/grpc/test/grpc_testing"
"google.golang.org/grpc/xds"
"google.golang.org/grpc/xds/internal/env"
"google.golang.org/grpc/xds/internal/testutils"
"google.golang.org/grpc/xds/internal/testutils/fakeserver"
"google.golang.org/grpc/xds/internal/testutils/e2e"
"google.golang.org/grpc/xds/internal/version"
)

Expand All @@ -58,94 +58,36 @@ func Test(t *testing.T) {
grpctest.RunSubTests(t, s{})
}

func setupListenerResponse(respCh chan *fakeserver.Response, name string) {
respCh <- &fakeserver.Response{
Resp: &v2discoverypb.DiscoveryResponse{
Resources: []*anypb.Any{
{
TypeUrl: version.V2ListenerURL,
Value: func() []byte {
l := &v3listenerpb.Listener{
// This needs to match the name we are querying for.
Name: name,
ApiListener: &v3listenerpb.ApiListener{
ApiListener: &anypb.Any{
TypeUrl: version.V2HTTPConnManagerURL,
Value: func() []byte {
cm := &v3httppb.HttpConnectionManager{
RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
Rds: &v3httppb.Rds{
ConfigSource: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
},
RouteConfigName: "route-config",
},
},
}
mcm, _ := proto.Marshal(cm)
return mcm
}(),
},
},
}
ml, _ := proto.Marshal(l)
return ml
}(),
},
},
TypeUrl: version.V2ListenerURL,
},
}
}

func setupBootstrapFile(t *testing.T, serverURI string) func() {
// Create a bootstrap file in a temporary directory.
tmpdir, err := ioutil.TempDir("", "xds-server-test*")
if err != nil {
t.Fatalf("failed to create tempdir: %v", err)
}
bootstrapContents := fmt.Sprintf(`
{
"node": {
"id": "ENVOY_NODE_ID",
"metadata": {
"TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
}
},
"xds_servers" : [{
"server_uri": "%s",
"channel_creds": [
{ "type": "insecure" }
]
}]
}`, serverURI)
bootstrapFileName := path.Join(tmpdir, "bootstrap")
if err := ioutil.WriteFile(bootstrapFileName, []byte(bootstrapContents), os.ModePerm); err != nil {
t.Fatalf("failed to write bootstrap file: %v", err)
}

origBootstrapFileName := env.BootstrapFileName
env.BootstrapFileName = bootstrapFileName
t.Logf("Create bootstrap file at %s with contents\n%s", bootstrapFileName, bootstrapContents)
return func() { env.BootstrapFileName = origBootstrapFileName }
}

// TestServerSideXDS is an e2e tests for xDS use on the server. This does not
// use any xDS features because we have not implemented any on the server side.
func (s) TestServerSideXDS(t *testing.T) {
// Spin up a fake xDS management server on a local port.
// TODO(easwars): Switch to using the server from envoy-go-control-plane.
fs, cleanup, err := fakeserver.StartServer()
origV3Support := env.V3Support
env.V3Support = true
defer func() { env.V3Support = origV3Support }()

ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()

// Spin up a xDS management server on a local port.
nodeID := uuid.New().String()
fs, err := e2e.StartManagementServer()
if err != nil {
t.Fatalf("failed to start fake xDS server: %v", err)
t.Fatal(err)
}
defer cleanup()
t.Logf("Started xDS management server at %s", fs.Address)
defer fs.Stop()

// Create a bootstrap file in a temporary directory.
defer setupBootstrapFile(t, fs.Address)()
cleanup, err := e2e.SetupBootstrapFile(e2e.BootstrapOptions{
Version: e2e.TransportV3,
NodeID: nodeID,
ServerURI: fs.Address,
})
if err != nil {
t.Fatal(err)
}
defer cleanup()

// Initialize a gRPC server which uses xDS, and register stubServer on it.
// Initialize an xDS-enabled gRPC server and register the stubServer on it.
server := xds.NewGRPCServer()
testpb.RegisterTestServiceServer(server, &testService{})
defer server.Stop()
Expand All @@ -162,18 +104,40 @@ func (s) TestServerSideXDS(t *testing.T) {
}
}()

// Create a ClientConn and make a successful RPC.
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
defer cancel()

// Setup the fake management server to respond with a Listener resource.
go func() {
if _, err := fs.XDSRequestChan.Receive(ctx); err != nil {
t.Errorf("timeout when waiting for listener request: %v", err)
listener := &v3listenerpb.Listener{
// This needs to match the name we are querying for.
Name: fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr),
ApiListener: &v3listenerpb.ApiListener{
ApiListener: &anypb.Any{
TypeUrl: version.V2HTTPConnManagerURL,
Value: func() []byte {
cm := &v3httppb.HttpConnectionManager{
RouteSpecifier: &v3httppb.HttpConnectionManager_Rds{
Rds: &v3httppb.Rds{
ConfigSource: &v3corepb.ConfigSource{
ConfigSourceSpecifier: &v3corepb.ConfigSource_Ads{Ads: &v3corepb.AggregatedConfigSource{}},
},
RouteConfigName: "route-config",
},
},
}
mcm, _ := proto.Marshal(cm)
return mcm
}(),
},
},
}
if err := fs.Update(e2e.UpdateOptions{
NodeID: nodeID,
Listeners: []*v3listenerpb.Listener{listener},
}); err != nil {
t.Error(err)
}
setupListenerResponse(fs.XDSResponseChan, fmt.Sprintf("grpc/server?udpa.resource.listening_address=%s", localAddr))
}()

// Create a ClientConn and make a successful RPC.
cc, err := grpc.DialContext(ctx, localAddr, grpc.WithTransportCredentials(insecure.NewCredentials()))
if err != nil {
t.Fatalf("failed to dial local test server: %v", err)
Expand Down
123 changes: 123 additions & 0 deletions xds/internal/testutils/e2e/bootstrap.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
/*
*
* Copyright 2020 gRPC authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*
*/

package e2e

import (
"encoding/json"
"fmt"
"io/ioutil"
"os"

"google.golang.org/grpc/xds/internal/env"
)

// TransportAPI refers to the API version for xDS transport protocol.
type TransportAPI int

const (
// TransportV2 refers to the v2 xDS transport protocol.
TransportV2 TransportAPI = iota
// TransportV3 refers to the v3 xDS transport protocol.
TransportV3
)

// BootstrapOptions wraps the parameters passed to SetupBootstrapFile.
type BootstrapOptions struct {
// Version is the xDS transport protocol version.
Version TransportAPI
// NodeID is the node identifier of the gRPC client/server node in the
// proxyless service mesh.
NodeID string
// ServerURI is the address of the management server.
ServerURI string
}

// SetupBootstrapFile creates a temporary file with bootstrap contents, based on
// the passed in options, and updates the bootstrap environment variable to
// point to this file.
//
// Returns a cleanup function which will be non-nil if the setup process was
// completed successfully. It is the responsibility of the caller to invoke the
// cleanup function at the end of the test.
func SetupBootstrapFile(opts BootstrapOptions) (func(), error) {
f, err := ioutil.TempFile("", "test_xds_bootstrap_*")
if err != nil {
return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
}

cfg := &bootstrapConfig{
XdsServers: []server{
{
ServerURI: opts.ServerURI,
ChannelCreds: []creds{
{
Type: "insecure",
},
},
},
},
Node: node{
ID: opts.NodeID,
},
}
switch opts.Version {
case TransportV2:
// TODO: Add any v2 specific fields.
case TransportV3:
cfg.ServerFeatures = append(cfg.ServerFeatures, "xds_v3")
default:
return nil, fmt.Errorf("unsupported xDS transport protocol version: %v", opts.Version)
}

bootstrapContents, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
}
if err := ioutil.WriteFile(f.Name(), bootstrapContents, 0644); err != nil {
return nil, fmt.Errorf("failed to created bootstrap file: %v", err)
}
logger.Infof("Created bootstrap file at %q with contents: %s\n", f.Name(), bootstrapContents)

origBootstrapFileName := env.BootstrapFileName
env.BootstrapFileName = f.Name()
return func() {
os.Remove(f.Name())
env.BootstrapFileName = origBootstrapFileName
}, nil
}

type bootstrapConfig struct {
XdsServers []server `json:"xds_servers,omitempty"`
Node node `json:"node,omitempty"`
ServerFeatures []string `json:"server_features,omitempty"`
}

type server struct {
ServerURI string `json:"server_uri,omitempty"`
ChannelCreds []creds `json:"channel_creds,omitempty"`
}

type creds struct {
Type string `json:"type,omitempty"`
Config interface{} `json:"config,omitempty"`
}

type node struct {
ID string `json:"id,omitempty"`
}

0 comments on commit 230166b

Please sign in to comment.