agent: Use an in-process listener with cache (#12762)

Uses a bufconn listener between consul-template and vault-agent when
caching is enabled and either templates or a listener is defined. This
means no listeners need to be defined in vault-agent for just
templating. Always routes consul-template through the vault-agent
cache (instead of only when persistent cache is enabled).

Uses a local transportDialer interface in config.Cache{}. 

Co-authored-by: Tom Proctor <tomhjp@users.noreply.github.com>
Co-authored-by: Ben Ash <32777270+benashz@users.noreply.github.com>

changelog/12762.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+agent/cache: Use an in-process listener between consul-template and vault-agent when caching is enabled and either templates or a listener is defined
+```

command/agent.go

@@ -2,6 +2,7 @@ package command
 
 import (
 	"context"
+	"crypto/tls"
 	"flag"
 	"fmt"
 	"io"
@@ -40,6 +41,8 @@ import (
 	"github.com/hashicorp/vault/command/agent/sink/inmem"
 	"github.com/hashicorp/vault/command/agent/template"
 	"github.com/hashicorp/vault/command/agent/winsvc"
+	"github.com/hashicorp/vault/internalshared/configutil"
+	"github.com/hashicorp/vault/internalshared/listenerutil"
 	"github.com/hashicorp/vault/sdk/helper/consts"
 	"github.com/hashicorp/vault/sdk/helper/logging"
 	"github.com/hashicorp/vault/sdk/logical"
@@ -48,6 +51,7 @@ import (
 	"github.com/mitchellh/cli"
 	"github.com/oklog/run"
 	"github.com/posener/complete"
+	"google.golang.org/grpc/test/bufconn"
 )
 
 var (
@@ -470,7 +474,7 @@ func (c *AgentCommand) Run(args []string) int {
 	var leaseCache *cache.LeaseCache
 	var previousToken string
 	// Parse agent listener configurations
-	if config.Cache != nil && len(config.Listeners) != 0 {
+	if config.Cache != nil {
 		cacheLogger := c.logger.Named("cache")
 
 		// Create the API proxier
@@ -666,11 +670,25 @@ func (c *AgentCommand) Run(args []string) int {
 		cacheHandler := cache.Handler(ctx, cacheLogger, leaseCache, inmemSink, proxyVaultToken)
 
 		var listeners []net.Listener
+
+		// If there are templates, add an in-process listener
+		if len(config.Templates) > 0 {
+			config.Listeners = append(config.Listeners, &configutil.Listener{Type: listenerutil.BufConnType})
+		}
 		for i, lnConfig := range config.Listeners {
-			ln, tlsConf, err := cache.StartListener(lnConfig)
-			if err != nil {
-				c.UI.Error(fmt.Sprintf("Error starting listener: %v", err))
-				return 1
+			var ln net.Listener
+			var tlsConf *tls.Config
+
+			if lnConfig.Type == listenerutil.BufConnType {
+				inProcListener := bufconn.Listen(1024 * 1024)
+				config.Cache.InProcDialer = listenerutil.NewBufConnWrapper(inProcListener)
+				ln = inProcListener
+			} else {
+				ln, tlsConf, err = cache.StartListener(lnConfig)
+				if err != nil {
+					c.UI.Error(fmt.Sprintf("Error starting listener: %v", err))
+					return 1
+				}
 			}
 
 			listeners = append(listeners, ln)

command/agent/config/config.go

@@ -1,9 +1,11 @@
 package config
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io/ioutil"
+	"net"
 	"os"
 	"strings"
 	"time"
@@ -64,14 +66,25 @@ type Vault struct {
 	Retry            *Retry      `hcl:"retry"`
 }
 
+// transportDialer is an interface that allows passing a custom dialer function
+// to an HTTP client's transport config
+type transportDialer interface {
+	// Dial is intended to match https://pkg.go.dev/net#Dialer.Dial
+	Dial(network, address string) (net.Conn, error)
+
+	// DialContext is intended to match https://pkg.go.dev/net#Dialer.DialContext
+	DialContext(ctx context.Context, network, address string) (net.Conn, error)
+}
+
 // Cache contains any configuration needed for Cache mode
 type Cache struct {
-	UseAutoAuthTokenRaw interface{} `hcl:"use_auto_auth_token"`
-	UseAutoAuthToken    bool        `hcl:"-"`
-	ForceAutoAuthToken  bool        `hcl:"-"`
-	EnforceConsistency  string      `hcl:"enforce_consistency"`
-	WhenInconsistent    string      `hcl:"when_inconsistent"`
-	Persist             *Persist    `hcl:"persist"`
+	UseAutoAuthTokenRaw interface{}     `hcl:"use_auto_auth_token"`
+	UseAutoAuthToken    bool            `hcl:"-"`
+	ForceAutoAuthToken  bool            `hcl:"-"`
+	EnforceConsistency  string          `hcl:"enforce_consistency"`
+	WhenInconsistent    string          `hcl:"when_inconsistent"`
+	Persist             *Persist        `hcl:"persist"`
+	InProcDialer        transportDialer `hcl:"-"`
 }
 
 // Persist contains configuration needed for persistent caching
@@ -203,8 +216,8 @@ func LoadConfig(path string) (*Config, error) {
 	}
 
 	if result.Cache != nil {
-		if len(result.Listeners) < 1 {
-			return nil, fmt.Errorf("at least one listener required when cache enabled")
+		if len(result.Listeners) < 1 && len(result.Templates) < 1 {
+			return nil, fmt.Errorf("enabling the cache requires at least 1 template or 1 listener to be defined")
 		}
 
 		if result.Cache.UseAutoAuthToken {

command/agent/config/config_test.go

@@ -105,6 +105,74 @@ func TestLoadConfigFile_AgentCache(t *testing.T) {
 	}
 }
 
+func TestLoadConfigFile_AgentCache_NoListeners(t *testing.T) {
+	config, err := LoadConfig("./test-fixtures/config-cache-no-listeners.hcl")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := &Config{
+		SharedConfig: &configutil.SharedConfig{
+			PidFile: "./pidfile",
+		},
+		AutoAuth: &AutoAuth{
+			Method: &Method{
+				Type:      "aws",
+				MountPath: "auth/aws",
+				Config: map[string]interface{}{
+					"role": "foobar",
+				},
+			},
+			Sinks: []*Sink{
+				{
+					Type:   "file",
+					DHType: "curve25519",
+					DHPath: "/tmp/file-foo-dhpath",
+					AAD:    "foobar",
+					Config: map[string]interface{}{
+						"path": "/tmp/file-foo",
+					},
+				},
+			},
+		},
+		Cache: &Cache{
+			UseAutoAuthToken:    true,
+			UseAutoAuthTokenRaw: true,
+			ForceAutoAuthToken:  false,
+			Persist: &Persist{
+				Type:                    "kubernetes",
+				Path:                    "/vault/agent-cache/",
+				KeepAfterImport:         true,
+				ExitOnErr:               true,
+				ServiceAccountTokenFile: "/tmp/serviceaccount/token",
+			},
+		},
+		Vault: &Vault{
+			Address:          "http://127.0.0.1:1111",
+			CACert:           "config_ca_cert",
+			CAPath:           "config_ca_path",
+			TLSSkipVerifyRaw: interface{}("true"),
+			TLSSkipVerify:    true,
+			ClientCert:       "config_client_cert",
+			ClientKey:        "config_client_key",
+			Retry: &Retry{
+				NumRetries: 12,
+			},
+		},
+		Templates: []*ctconfig.TemplateConfig{
+			{
+				Source:      pointerutil.StringPtr("/path/on/disk/to/template.ctmpl"),
+				Destination: pointerutil.StringPtr("/path/on/disk/where/template/will/render.txt"),
+			},
+		},
+	}
+
+	config.Prune()
+	if diff := deep.Equal(config, expected); diff != nil {
+		t.Fatal(diff)
+	}
+}
+
 func TestLoadConfigFile(t *testing.T) {
 	if err := os.Setenv("TEST_AAD_ENV", "aad"); err != nil {
 		t.Fatal(err)
@@ -270,7 +338,7 @@ func TestLoadConfigFile_Bad_AgentCache_ForceAutoAuthNoMethod(t *testing.T) {
 func TestLoadConfigFile_Bad_AgentCache_NoListeners(t *testing.T) {
 	_, err := LoadConfig("./test-fixtures/bad-config-cache-no-listeners.hcl")
 	if err == nil {
-		t.Fatal("LoadConfig should return an error when cache section present and no listeners present")
+		t.Fatal("LoadConfig should return an error when cache section present and no listeners present and no templates defined")
 	}
 }
 

command/agent/config/test-fixtures/config-cache-no-listeners.hcl

@@ -0,0 +1,45 @@
+pid_file = "./pidfile"
+
+auto_auth {
+	method {
+		type = "aws"
+		config = {
+			role = "foobar"
+		}
+	}
+
+	sink {
+		type = "file"
+		config = {
+			path = "/tmp/file-foo"
+		}
+		aad = "foobar"
+		dh_type = "curve25519"
+		dh_path = "/tmp/file-foo-dhpath"
+	}
+}
+
+cache {
+	use_auto_auth_token = true
+	persist = {
+		type = "kubernetes"
+		path = "/vault/agent-cache/"
+		keep_after_import = true
+		exit_on_err = true
+		service_account_token_file = "/tmp/serviceaccount/token"
+	}
+}
+
+vault {
+	address = "http://127.0.0.1:1111"
+	ca_cert = "config_ca_cert"
+	ca_path = "config_ca_path"
+	tls_skip_verify = "true"
+	client_cert = "config_client_cert"
+	client_key = "config_client_key"
+}
+
+template {
+  source      = "/path/on/disk/to/template.ctmpl"
+  destination = "/path/on/disk/where/template/will/render.txt"
+}

command/agent/template/template.go

@@ -264,10 +264,7 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) (*ctc
 	}
 
 	// Use the cache if available or fallback to the Vault server values.
-	// For now we're only routing templating through the cache when persistence
-	// is enabled. The templating engine and the cache have some inconsistencies
-	// that need to be fixed for 1.7x/1.8
-	if sc.AgentConfig.Cache != nil && sc.AgentConfig.Cache.Persist != nil && len(sc.AgentConfig.Listeners) != 0 {
+	if sc.AgentConfig.Cache != nil {
 		attempts = 0
 
 		// If we don't want exit on template retry failure (i.e. unlimited
@@ -283,23 +280,18 @@ func newRunnerConfig(sc *ServerConfig, templates ctconfig.TemplateConfigs) (*ctc
 			attempts = ctconfig.DefaultRetryAttempts
 		}
 
-		scheme := "unix://"
-		if sc.AgentConfig.Listeners[0].Type == "tcp" {
-			scheme = "https://"
-			if sc.AgentConfig.Listeners[0].TLSDisable {
-				scheme = "http://"
-			}
+		if sc.AgentConfig.Cache.InProcDialer == nil {
+			return nil, fmt.Errorf("missing in-process dialer configuration")
 		}
-		address := fmt.Sprintf("%s%s", scheme, sc.AgentConfig.Listeners[0].Address)
-		conf.Vault.Address = &address
-
-		// Skip verification if its using the cache because they're part of the same agent.
-		if scheme == "https://" {
-			if sc.AgentConfig.Listeners[0].TLSRequireAndVerifyClientCert {
-				return nil, errors.New("template server cannot use local cache when mTLS is enabled")
-			}
-			conf.Vault.SSL.Verify = pointerutil.BoolPtr(false)
+		if conf.Vault.Transport == nil {
+			conf.Vault.Transport = &ctconfig.TransportConfig{}
 		}
+		conf.Vault.Transport.CustomDialer = sc.AgentConfig.Cache.InProcDialer
+		// The in-process dialer ignores the address passed in, but we're still
+		// setting it here to override the setting at the top of this function,
+		// and to prevent the vault/http client from defaulting to https.
+		conf.Vault.Address = pointerutil.StringPtr("http://127.0.0.1:8200")
+
 	} else if strings.HasPrefix(sc.AgentConfig.Vault.Address, "https") || sc.AgentConfig.Vault.CACert != "" {
 		skipVerify := sc.AgentConfig.Vault.TLSSkipVerify
 		verify := !skipVerify