xds: Add bootstrap support for certificate providers. (#3901)

credentials/tls/certprovider/provider.go

@@ -29,8 +29,14 @@ import (
 	"crypto/tls"
 	"crypto/x509"
 	"errors"
+
+	"google.golang.org/grpc/internal"
 )
 
+func init() {
+	internal.GetCertificateProviderBuilder = getBuilder
+}
+
 var (
 	// errProviderClosed is returned by Distributor.KeyMaterial when it is
 	// closed.

internal/internal.go

@@ -52,6 +52,11 @@ var (
 	// This function compares the config without rawJSON stripped, in case the
 	// there's difference in white space.
 	EqualServiceConfigForTesting func(a, b serviceconfig.Config) bool
+	// GetCertificateProviderBuilder returns the registered builder for the
+	// given name. This is set by package certprovider for use from xDS
+	// bootstrap code while parsing certificate provider configs in the
+	// bootstrap file.
+	GetCertificateProviderBuilder interface{} // func(string) certprovider.Builder
 )
 
 // HealthChecker defines the signature of the client-side LB channel health checking function.

xds/internal/client/bootstrap/bootstrap.go

@@ -33,6 +33,8 @@ import (
 	"github.com/golang/protobuf/proto"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -77,6 +79,18 @@ type Config struct {
 	// NodeProto contains the Node proto to be used in xDS requests. The actual
 	// type depends on the transport protocol version used.
 	NodeProto proto.Message
+	// CertProviderConfigs contain parsed configs for supported certificate
+	// provider plugins found in the bootstrap file.
+	CertProviderConfigs map[string]CertProviderConfig
+}
+
+// CertProviderConfig wraps the certificate provider plugin name and config
+// (corresponding to one plugin instance) found in the bootstrap file.
+type CertProviderConfig struct {
+	// Name is the registered name of the certificate provider.
+	Name string
+	// Config is the parsed config to be passed to the certificate provider.
+	Config certprovider.StableConfig
 }
 
 type channelCreds struct {
@@ -103,6 +117,10 @@ type xdsServer struct {
 //        }
 //      ],
 //      "server_features": [ ... ]
+//		"certificate_providers" : {
+//			"default": { default cert provider config },
+//			"foo": { config for provider foo }
+//		}
 //    },
 //    "node": <JSON form of Node proto>
 // }
@@ -182,6 +200,35 @@ func NewConfig() (*Config, error) {
 					serverSupportsV3 = true
 				}
 			}
+		case "certificate_providers":
+			var providerInstances map[string]json.RawMessage
+			if err := json.Unmarshal(v, &providerInstances); err != nil {
+				return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), k, err)
+			}
+			configs := make(map[string]CertProviderConfig)
+			getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
+			for instance, data := range providerInstances {
+				var providerConfigs map[string]json.RawMessage
+				if err := json.Unmarshal(data, &providerConfigs); err != nil {
+					return nil, fmt.Errorf("xds: json.Unmarshal(%v) for field %q failed during bootstrap: %v", string(v), instance, err)
+				}
+				for name, cfg := range providerConfigs {
+					parser := getBuilder(name)
+					if parser == nil {
+						// We ignore plugins that we do not know about.
+						continue
+					}
+					c, err := parser.ParseConfig(cfg)
+					if err != nil {
+						return nil, fmt.Errorf("xds: Config parsing for plugin %q failed: %v", name, err)
+					}
+					configs[instance] = CertProviderConfig{
+						Name:   name,
+						Config: c,
+					}
+				}
+			}
+			config.CertProviderConfigs = configs
 		}
 		// Do not fail the xDS bootstrap when an unknown field is seen. This can
 		// happen when an older version client reads a newer version bootstrap

xds/internal/client/bootstrap/bootstrap_test.go

@@ -19,6 +19,8 @@
 package bootstrap
 
 import (
+	"encoding/json"
+	"errors"
 	"fmt"
 	"os"
 	"testing"
@@ -28,8 +30,11 @@ import (
 	"github.com/golang/protobuf/proto"
 	structpb "github.com/golang/protobuf/ptypes/struct"
 	"github.com/google/go-cmp/cmp"
+
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/google"
+	"google.golang.org/grpc/credentials/tls/certprovider"
+	"google.golang.org/grpc/internal"
 	"google.golang.org/grpc/xds/internal/version"
 )
 
@@ -233,6 +238,24 @@ func (c *Config) compare(want *Config) error {
 	if diff := cmp.Diff(want.NodeProto, c.NodeProto, cmp.Comparer(proto.Equal)); diff != "" {
 		return fmt.Errorf("config.NodeProto diff (-want, +got):\n%s", diff)
 	}
+
+	// A vanilla cmp.Equal or cmp.Diff will not produce useful error message
+	// here. So, we iterate through the list of configs and compare them one at
+	// a time.
+	gotCfgs := c.CertProviderConfigs
+	wantCfgs := want.CertProviderConfigs
+	if len(gotCfgs) != len(wantCfgs) {
+		return fmt.Errorf("config.CertProviderConfigs is %d entries, want %d", len(gotCfgs), len(wantCfgs))
+	}
+	for instance, gotCfg := range gotCfgs {
+		wantCfg, ok := wantCfgs[instance]
+		if !ok {
+			return fmt.Errorf("config.CertProviderConfigs has unexpected plugin instance %q with config %q", instance, string(gotCfg.Config.Canonical()))
+		}
+		if gotCfg.Name != wantCfg.Name || !cmp.Equal(gotCfg.Config.Canonical(), wantCfg.Config.Canonical()) {
+			return fmt.Errorf("config.CertProviderConfigs for plugin instance %q has config {%s, %s, want {%s, %s}", instance, gotCfg.Name, string(gotCfg.Config.Canonical()), wantCfg.Name, string(wantCfg.Config.Canonical()))
+		}
+	}
 	return nil
 }
 
@@ -452,3 +475,231 @@ func TestNewConfigBootstrapFileEnvNotSet(t *testing.T) {
 		t.Errorf("NewConfig() returned nil error, expected to fail")
 	}
 }
+
+func init() {
+	certprovider.Register(&fakeCertProviderBuilder{})
+}
+
+const fakeCertProviderName = "fake-certificate-provider"
+
+// fakeCertProviderBuilder builds new instances of fakeCertProvider and
+// interprets the config provided to it as JSON with a single key and value.
+type fakeCertProviderBuilder struct{}
+
+func (b *fakeCertProviderBuilder) Build(certprovider.StableConfig, certprovider.Options) certprovider.Provider {
+	return &fakeCertProvider{}
+}
+
+// ParseConfig expects input in JSON format containing a map from string to
+// string, with a single entry and mapKey being "configKey".
+func (b *fakeCertProviderBuilder) ParseConfig(cfg interface{}) (certprovider.StableConfig, error) {
+	config, ok := cfg.(json.RawMessage)
+	if !ok {
+		return nil, fmt.Errorf("fakeCertProviderBuilder received config of type %T, want []byte", config)
+	}
+	var cfgData map[string]string
+	if err := json.Unmarshal(config, &cfgData); err != nil {
+		return nil, fmt.Errorf("fakeCertProviderBuilder config parsing failed: %v", err)
+	}
+	if len(cfgData) != 1 || cfgData["configKey"] == "" {
+		return nil, errors.New("fakeCertProviderBuilder received invalid config")
+	}
+	return &fakeStableConfig{config: cfgData}, nil
+}
+
+func (b *fakeCertProviderBuilder) Name() string {
+	return fakeCertProviderName
+}
+
+type fakeStableConfig struct {
+	config map[string]string
+}
+
+func (c *fakeStableConfig) Canonical() []byte {
+	var cfg string
+	for k, v := range c.config {
+		cfg = fmt.Sprintf("%s:%s", k, v)
+	}
+	return []byte(cfg)
+}
+
+// fakeCertProvider is an empty implementation of the Provider interface.
+type fakeCertProvider struct {
+	certprovider.Provider
+}
+
+func TestNewConfigWithCertificateProviders(t *testing.T) {
+	bootstrapFileMap := map[string]string{
+		"badJSONCertProviderConfig": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": "bad JSON"
+		}`,
+		"allUnknownCertProviders": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": {
+				"unknownProviderInstance1": {
+					"foo1": "bar1"
+				},
+				"unknownProviderInstance2": {
+					"foo2": "bar2"
+				}
+			}
+		}`,
+		"badCertProviderConfig": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": {
+				"unknownProviderInstance": {
+					"foo": "bar"
+				},
+				"fakeProviderInstance": {
+					"fake-certificate-provider": {
+						"configKey": "configValue"
+					}
+				},
+				"fakeProviderInstanceBad": {
+					"fake-certificate-provider": {
+						"configKey": 666
+					}
+				}
+			}
+		}`,
+		"goodCertProviderConfig": `
+		{
+			"node": {
+				"id": "ENVOY_NODE_ID",
+				"metadata": {
+				    "TRAFFICDIRECTOR_GRPC_HOSTNAME": "trafficdirector"
+			    }
+			},
+			"xds_servers" : [{
+				"server_uri": "trafficdirector.googleapis.com:443",
+				"channel_creds": [
+					{ "type": "google_default" }
+				]
+			}],
+			"server_features" : ["foo", "bar", "xds_v3"],
+			"certificate_providers": {
+				"unknownProviderInstance": {
+					"foo": "bar"
+				},
+				"fakeProviderInstance": {
+					"fake-certificate-provider": {
+						"configKey": "configValue"
+					}
+				}
+			}
+		}`,
+	}
+
+	getBuilder := internal.GetCertificateProviderBuilder.(func(string) certprovider.Builder)
+	parser := getBuilder(fakeCertProviderName)
+	if parser == nil {
+		t.Fatalf("missing certprovider plugin %q", fakeCertProviderName)
+	}
+	wantCfg, err := parser.ParseConfig(json.RawMessage(`{"configKey": "configValue"}`))
+	if err != nil {
+		t.Fatalf("config parsing for plugin %q failed: %v", fakeCertProviderName, err)
+	}
+
+	if err := os.Setenv(v3SupportEnv, "true"); err != nil {
+		t.Fatalf("os.Setenv(%s, %s) failed with error: %v", v3SupportEnv, "true", err)
+	}
+	defer os.Unsetenv(v3SupportEnv)
+
+	cancel := setupBootstrapOverride(bootstrapFileMap)
+	defer cancel()
+
+	goodConfig := &Config{
+		BalancerName: "trafficdirector.googleapis.com:443",
+		Creds:        grpc.WithCredentialsBundle(google.NewComputeEngineCredentials()),
+		TransportAPI: version.TransportV3,
+		NodeProto:    v3NodeProto,
+		CertProviderConfigs: map[string]CertProviderConfig{
+			"fakeProviderInstance": {
+				Name:   fakeCertProviderName,
+				Config: wantCfg,
+			},
+		},
+	}
+	tests := []struct {
+		name       string
+		wantConfig *Config
+		wantErr    bool
+	}{
+		{
+			name:    "badJSONCertProviderConfig",
+			wantErr: true,
+		},
+		{
+
+			name:    "badCertProviderConfig",
+			wantErr: true,
+		},
+		{
+
+			name:       "allUnknownCertProviders",
+			wantConfig: nonNilCredsConfigV3,
+		},
+		{
+			name:       "goodCertProviderConfig",
+			wantConfig: goodConfig,
+		},
+	}
+
+	for _, test := range tests {
+		t.Run(test.name, func(t *testing.T) {
+			if err := os.Setenv(bootstrapFileEnv, test.name); err != nil {
+				t.Fatalf("os.Setenv(%s, %s) failed with error: %v", bootstrapFileEnv, test.name, err)
+			}
+			c, err := NewConfig()
+			if (err != nil) != test.wantErr {
+				t.Fatalf("NewConfig() returned: %v, wantErr: %v", err, test.wantErr)
+			}
+			if test.wantErr {
+				return
+			}
+			if err := c.compare(test.wantConfig); err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}