Customizing HTTP headers in the config file (#12485)

* Customizing HTTP headers in the config file

* Add changelog, fix bad imports

* fixing some bugs

* fixing interaction of custom headers and /ui

* Defining a member in core to set custom response headers

* missing additional file

* Some refactoring

* Adding automated tests for the feature

* Changing some error messages based on some recommendations

* Incorporating custom response headers struct into the request context

* removing some unused references

* fixing a test

* changing some error messages, removing a default header value from /ui

* fixing a test

* wrapping ResponseWriter to set the custom headers

* adding a new test

* some cleanup

* removing some extra lines

* Addressing comments

* fixing some agent tests

* skipping custom headers from agent listener config,
removing two of the default headers as they cause issues with Vault in UI mode
Adding X-Content-Type-Options to the ui default headers
Let Content-Type be set as before

* Removing default custom headers, and renaming some function varibles

* some refacotring

* Refactoring and addressing comments

* removing a function and fixing comments

changelog/12485.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+**Customizable HTTP Headers**: Add support to define custom HTTP headers for root path (`/`) and also on API endpoints (`/v1/*`)
+```

command/agent/config/config.go

@@ -35,6 +35,7 @@ func (c *Config) Prune() {
 		l.RawConfig = nil
 		l.Profiling.UnusedKeys = nil
 		l.Telemetry.UnusedKeys = nil
+		l.CustomResponseHeaders = nil
 	}
 	c.FoundKeys = nil
 	c.UnusedKeys = nil
@@ -172,6 +173,12 @@ func LoadConfig(path string) (*Config, error) {
 	if err != nil {
 		return nil, err
 	}
+
+	// Pruning custom headers for Agent for now
+	for _, ln := range sharedConfig.Listeners {
+		ln.CustomResponseHeaders = nil
+	}
+
 	result.SharedConfig = sharedConfig
 
 	list, ok := obj.Node.(*ast.ObjectList)

command/agent/config/config_test.go

@@ -536,7 +536,6 @@ func TestLoadConfigFile_AgentCache_PersistMissingType(t *testing.T) {
 }
 
 func TestLoadConfigFile_TemplateConfig(t *testing.T) {
-
 	testCases := map[string]struct {
 		fixturePath            string
 		expectedTemplateConfig TemplateConfig
@@ -586,7 +585,6 @@ func TestLoadConfigFile_TemplateConfig(t *testing.T) {
 			}
 		})
 	}
-
 }
 
 // TestLoadConfigFile_Template tests template definitions in Vault Agent

command/server.go

@@ -1541,6 +1541,12 @@ func (c *ServerCommand) Run(args []string) int {
 
 			core.SetConfig(config)
 
+			// reloading custom response headers to make sure we have
+			// the most up to date headers after reloading the config file
+			if err = core.ReloadCustomResponseHeaders(); err != nil {
+				c.logger.Error(err.Error())
+			}
+
 			if config.LogLevel != "" {
 				configLogLevel := strings.ToLower(strings.TrimSpace(config.LogLevel))
 				switch configLogLevel {

command/server/config_custom_response_headers_test.go

@@ -0,0 +1,109 @@
+package server
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/go-test/deep"
+)
+
+var defaultCustomHeaders = map[string]string{
+	"Strict-Transport-Security": "max-age=1; domains",
+	"Content-Security-Policy":   "default-src 'others'",
+	"X-Vault-Ignored":           "ignored",
+	"X-Custom-Header":           "Custom header value default",
+}
+
+var customHeaders307 = map[string]string{
+	"X-Custom-Header": "Custom header value 307",
+}
+
+var customHeader3xx = map[string]string{
+	"X-Vault-Ignored-3xx": "Ignored 3xx",
+	"X-Custom-Header":     "Custom header value 3xx",
+}
+
+var customHeaders200 = map[string]string{
+	"Someheader-200":  "200",
+	"X-Custom-Header": "Custom header value 200",
+}
+
+var customHeader2xx = map[string]string{
+	"X-Custom-Header": "Custom header value 2xx",
+}
+
+var customHeader400 = map[string]string{
+	"Someheader-400": "400",
+}
+
+var defaultCustomHeadersMultiListener = map[string]string{
+	"Strict-Transport-Security": "max-age=31536000; includeSubDomains",
+	"Content-Security-Policy":   "default-src 'others'",
+	"X-Vault-Ignored":           "ignored",
+	"X-Custom-Header":           "Custom header value default",
+}
+
+var defaultSTS = map[string]string{
+	"Strict-Transport-Security": "max-age=31536000; includeSubDomains",
+}
+
+func TestCustomResponseHeadersConfigs(t *testing.T) {
+	expectedCustomResponseHeader := map[string]map[string]string{
+		"default": defaultCustomHeaders,
+		"307":     customHeaders307,
+		"3xx":     customHeader3xx,
+		"200":     customHeaders200,
+		"2xx":     customHeader2xx,
+		"400":     customHeader400,
+	}
+
+	config, err := LoadConfigFile("./test-fixtures/config_custom_response_headers_1.hcl")
+	if err != nil {
+		t.Fatalf("Error encountered when loading config %+v", err)
+	}
+	if diff := deep.Equal(expectedCustomResponseHeader, config.Listeners[0].CustomResponseHeaders); diff != nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+}
+
+func TestCustomResponseHeadersConfigsMultipleListeners(t *testing.T) {
+	expectedCustomResponseHeader := map[string]map[string]string{
+		"default": defaultCustomHeadersMultiListener,
+		"307":     customHeaders307,
+		"3xx":     customHeader3xx,
+		"200":     customHeaders200,
+		"2xx":     customHeader2xx,
+		"400":     customHeader400,
+	}
+
+	config, err := LoadConfigFile("./test-fixtures/config_custom_response_headers_multiple_listeners.hcl")
+	if err != nil {
+		t.Fatalf("Error encountered when loading config %+v", err)
+	}
+	if diff := deep.Equal(expectedCustomResponseHeader, config.Listeners[0].CustomResponseHeaders); diff != nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+
+	if diff := deep.Equal(expectedCustomResponseHeader, config.Listeners[1].CustomResponseHeaders); diff == nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+	if diff := deep.Equal(expectedCustomResponseHeader["default"], config.Listeners[1].CustomResponseHeaders["default"]); diff != nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+
+	if diff := deep.Equal(expectedCustomResponseHeader, config.Listeners[2].CustomResponseHeaders); diff == nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+
+	if diff := deep.Equal(defaultSTS, config.Listeners[2].CustomResponseHeaders["default"]); diff != nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+
+	if diff := deep.Equal(expectedCustomResponseHeader, config.Listeners[3].CustomResponseHeaders); diff == nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+
+	if diff := deep.Equal(defaultSTS, config.Listeners[3].CustomResponseHeaders["default"]); diff != nil {
+		t.Fatalf(fmt.Sprintf("parsed custom headers do not match the expected ones, difference: %v", diff))
+	}
+}

command/server/config_test_helpers.go

@@ -16,6 +16,12 @@ import (
 	"github.com/hashicorp/vault/internalshared/configutil"
 )
 
+var DefaultCustomHeaders = map[string]map[string]string {
+	"default": {
+		"Strict-Transport-Security": configutil.StrictTransportSecurity,
+	},
+}
+
 func boolPointer(x bool) *bool {
 	return &x
 }
@@ -32,6 +38,7 @@ func testConfigRaftRetryJoin(t *testing.T) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:8200",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 			DisableMlock: true,
@@ -64,6 +71,7 @@ func testLoadConfigFile_topLevel(t *testing.T, entropy *configutil.Entropy) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:443",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 
@@ -174,10 +182,12 @@ func testLoadConfigFile_json2(t *testing.T, entropy *configutil.Entropy) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:443",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:444",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 
@@ -336,6 +346,7 @@ func testLoadConfigFileIntegerAndBooleanValuesCommon(t *testing.T, path string)
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:8200",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 			DisableMlock: true,
@@ -379,6 +390,7 @@ func testLoadConfigFile(t *testing.T) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:443",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 
@@ -486,7 +498,7 @@ func testUnknownFieldValidation(t *testing.T) {
 	for _, er1 := range errors {
 		found := false
 		if strings.Contains(er1.String(), "sentinel") {
-			//This happens on OSS, and is fine
+			// This happens on OSS, and is fine
 			continue
 		}
 		for _, ex := range expected {
@@ -525,6 +537,7 @@ func testLoadConfigFile_json(t *testing.T) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:443",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 
@@ -610,6 +623,7 @@ func testLoadConfigDir(t *testing.T) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:443",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 
@@ -818,6 +832,7 @@ listener "tcp" {
 					Profiling: configutil.ListenerProfiling{
 						UnauthenticatedPProfAccess: true,
 					},
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 		},
@@ -845,6 +860,7 @@ func testParseSeals(t *testing.T) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:443",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 			Seals: []*configutil.KMS{
@@ -898,6 +914,7 @@ func testLoadConfigFileLeaseMetrics(t *testing.T) {
 				{
 					Type:    "tcp",
 					Address: "127.0.0.1:443",
+					CustomResponseHeaders: DefaultCustomHeaders,
 				},
 			},
 

command/server/test-fixtures/config_custom_response_headers_1.hcl

@@ -0,0 +1,31 @@
+storage "inmem" {}
+listener "tcp" {
+  address = "127.0.0.1:8200"
+  tls_disable = true
+  custom_response_headers  {
+    "default" = {
+       "Strict-Transport-Security" = ["max-age=1","domains"],
+       "Content-Security-Policy" = ["default-src 'others'"],
+       "X-Vault-Ignored" = ["ignored"],
+       "X-Custom-Header" = ["Custom header value default"],
+     }
+     "307" = {
+       "X-Custom-Header" = ["Custom header value 307"],
+     }
+     "3xx" = {
+       "X-Vault-Ignored-3xx" = ["Ignored 3xx"],
+       "X-Custom-Header" = ["Custom header value 3xx"]
+     }
+     "200" = {
+       "someheader-200" = ["200"],
+       "X-Custom-Header" = ["Custom header value 200"]
+     }
+     "2xx" = {
+       "X-Custom-Header" = ["Custom header value 2xx"]
+     }
+     "400" = {
+       "someheader-400" = ["400"]
+     }
+  }
+}
+disable_mlock = true

command/server/test-fixtures/config_custom_response_headers_multiple_listeners.hcl

@@ -0,0 +1,56 @@
+storage "inmem" {}
+listener "tcp" {
+  address = "127.0.0.1:8200"
+  tls_disable = true
+  custom_response_headers  {
+    "default" = {
+       "Content-Security-Policy" = ["default-src 'others'"],
+       "X-Vault-Ignored" = ["ignored"],
+       "X-Custom-Header" = ["Custom header value default"],
+     }
+     "307" = {
+       "X-Custom-Header" = ["Custom header value 307"],
+     }
+     "3xx" = {
+       "X-Vault-Ignored-3xx" = ["Ignored 3xx"],
+       "X-Custom-Header" = ["Custom header value 3xx"]
+     }
+     "200" = {
+       "someheader-200" = ["200"],
+       "X-Custom-Header" = ["Custom header value 200"]
+     }
+     "2xx" = {
+       "X-Custom-Header" = ["Custom header value 2xx"]
+     }
+     "400" = {
+       "someheader-400" = ["400"]
+     }
+  }
+}
+listener "tcp" {
+  address = "127.0.0.2:8200"
+  tls_disable = true
+  custom_response_headers  {
+    "default" = {
+       "Content-Security-Policy" = ["default-src 'others'"],
+       "X-Vault-Ignored" = ["ignored"],
+       "X-Custom-Header" = ["Custom header value default"],
+     }
+  }
+}
+listener "tcp" {
+  address = "127.0.0.3:8200"
+  tls_disable = true
+  custom_response_headers  {
+    "2xx" = {
+       "X-Custom-Header" = ["Custom header value 2xx"]
+    }
+  }
+}
+listener "tcp" {
+  address = "127.0.0.4:8200"
+  tls_disable = true
+}
+
+
+disable_mlock = true