[dev] Support request headers override in middleware (#8752)

Implements request headers override in middlewares.

#### New middleware headers

- `x-middleware-override-headers`: A comma separated list of *all* request header names. Headers not listed will be deleted.
- `x-middleware-request-<name>`: A new value for the header `<name>`.

### Related Issues

- #8724: Add helper functions for non-Next.js middlewares
- vercel/next.js#41380: Next.js' implementation

### 📋 Checklist

<!--
  Please keep your PR as a Draft until the checklist is complete
-->

#### Tests

- [ ] The code changed/added as part of this PR has been covered with tests
- [ ] All tests pass locally with `yarn test-unit`

#### Code Review

- [ ] This PR has a concise title and thorough description useful to a reviewer
- [ ] Issue from task tracker has a link to this PR

packages/cli/src/util/dev/headers.ts

@@ -16,3 +16,75 @@ export function nodeHeadersToFetchHeaders(
   }
   return headers;
 }
+
+/**
+ * Request headers that are not allowed to be overridden by a middleware.
+ */
+const NONOVERRIDABLE_HEADERS: Set<string> = new Set([
+  'host',
+  'connection',
+  'content-length',
+  'transfer-encoding',
+  'keep-alive',
+  'transfer-encoding',
+  'te',
+  'upgrade',
+  'trailer',
+]);
+
+/**
+ * Adds/Updates/Deletes headers in `reqHeaders` based on the response headers
+ * from a middleware (`respHeaders`).
+ *
+ * `x-middleware-override-headers` is a comma-separated list of *all* header
+ * names that should appear in new request headers. Names not in this list
+ * will be deleted.
+ *
+ * `x-middleware-request-*` is the new value for each header. This can't be
+ * omitted, even if the header is not being modified.
+ *
+ */
+export function applyOverriddenHeaders(
+  reqHeaders: { [k: string]: string | string[] | undefined },
+  respHeaders: Headers
+) {
+  const overriddenHeaders = respHeaders.get('x-middleware-override-headers');
+  if (!overriddenHeaders) {
+    return;
+  }
+
+  const overriddenKeys: Set<string> = new Set();
+  for (const key of overriddenHeaders.split(',')) {
+    overriddenKeys.add(key.trim());
+  }
+
+  respHeaders.delete('x-middleware-override-headers');
+
+  // Delete headers.
+  for (const key of Object.keys(reqHeaders)) {
+    if (!NONOVERRIDABLE_HEADERS.has(key) && !overriddenKeys.has(key)) {
+      delete reqHeaders[key];
+    }
+  }
+
+  // Update or add headers.
+  for (const key of overriddenKeys.keys()) {
+    if (NONOVERRIDABLE_HEADERS.has(key)) {
+      continue;
+    }
+
+    const valueKey = 'x-middleware-request-' + key;
+    const newValue = respHeaders.get(valueKey);
+    const oldValue = reqHeaders[key];
+
+    if (oldValue !== newValue) {
+      if (newValue) {
+        reqHeaders[key] = newValue;
+      } else {
+        delete reqHeaders[key];
+      }
+    }
+
+    respHeaders.delete(valueKey);
+  }
+}

packages/cli/src/util/dev/server.ts

@@ -87,7 +87,7 @@ import {
 } from './types';
 import { ProjectSettings } from '../../types';
 import { treeKill } from '../tree-kill';
-import { nodeHeadersToFetchHeaders } from './headers';
+import { applyOverriddenHeaders, nodeHeadersToFetchHeaders } from './headers';
 import { formatQueryString, parseQueryString } from './parse-query-string';
 import {
   errorToString,
@@ -1472,6 +1472,9 @@ export default class DevServer {
             'content-length',
             'transfer-encoding',
           ]);
+
+          applyOverriddenHeaders(req.headers, middlewareRes.headers);
+
           for (const [name, value] of middlewareRes.headers) {
             if (name === 'x-middleware-next') {
               shouldContinue = value === '1';

packages/cli/test/dev/fixtures/middleware-request-headers-override/api/dump-headers.js

@@ -0,0 +1,3 @@
+export default (req, res) => {
+  res.json(req.headers);
+};

packages/cli/test/dev/fixtures/middleware-request-headers-override/middleware.js

@@ -0,0 +1,18 @@
+export default () => {
+  return new Response(null, {
+    headers: {
+      'x-middleware-next': '1',
+      'x-middleware-override-headers':
+        'x-from-client-a,x-from-client-b,x-from-middleware-a,x-from-middleware-b,transfer-encoding',
+      // Headers to be preserved.
+      'x-middleware-request-x-from-client-a': 'hello from client',
+      // Headers to be modified by the middleware.
+      'x-middleware-request-x-from-client-b': 'hello from middleware',
+      // Headers to be added by the middleware.
+      'x-middleware-request-x-from-middleware-a': 'hello a!',
+      'x-middleware-request-x-from-middleware-b': 'hello b!',
+      // Headers not allowed by the dev server: will be ignored.
+      'transfer-encoding': 'gzip, chunked',
+    },
+  });
+};

packages/cli/test/dev/integration-4.test.ts

@@ -2,6 +2,7 @@ import ms from 'ms';
 import fs from 'fs-extra';
 import { isIP } from 'net';
 import { join } from 'path';
+import { Response } from 'node-fetch';
 
 const {
   fetch,
@@ -613,3 +614,72 @@ test(
     { skipDeploy: true }
   )
 );
+
+test(
+  '[vercel dev] Middleware can override request headers',
+  testFixtureStdio(
+    'middleware-request-headers-override',
+    async (testPath: any) => {
+      await testPath(
+        200,
+        '/api/dump-headers',
+        (actual: string, res: Response) => {
+          // Headers sent to the API route.
+          const headers = JSON.parse(actual);
+
+          // Preserved headers.
+          expect(headers).toHaveProperty(
+            'x-from-client-a',
+            'hello from client'
+          );
+
+          // Headers added/modified by the middleware.
+          expect(headers).toHaveProperty(
+            'x-from-client-b',
+            'hello from middleware'
+          );
+          expect(headers).toHaveProperty('x-from-middleware-a', 'hello a!');
+          expect(headers).toHaveProperty('x-from-middleware-b', 'hello b!');
+
+          // Headers deleted by the middleware.
+          expect(headers).not.toHaveProperty('x-from-client-c');
+
+          // Internal headers should not be visible from API routes.
+          expect(headers).not.toHaveProperty('x-middleware-override-headers');
+          expect(headers).not.toHaveProperty(
+            'x-middleware-request-from-middleware-a'
+          );
+          expect(headers).not.toHaveProperty(
+            'x-middleware-request-from-middleware-b'
+          );
+
+          // Request headers should not be visible from clients.
+          const respHeaders = Object.fromEntries(res.headers.entries());
+          expect(respHeaders).not.toHaveProperty(
+            'x-middleware-override-headers'
+          );
+          expect(respHeaders).not.toHaveProperty(
+            'x-middleware-request-from-middleware-a'
+          );
+          expect(respHeaders).not.toHaveProperty(
+            'x-middleware-request-from-middleware-b'
+          );
+          expect(respHeaders).not.toHaveProperty('from-middleware-a');
+          expect(respHeaders).not.toHaveProperty('from-middleware-b');
+          expect(respHeaders).not.toHaveProperty('x-from-client-a');
+          expect(respHeaders).not.toHaveProperty('x-from-client-b');
+          expect(respHeaders).not.toHaveProperty('x-from-client-c');
+        },
+        /*expectedHeaders=*/ {},
+        {
+          headers: {
+            'x-from-client-a': 'hello from client',
+            'x-from-client-b': 'hello from client',
+            'x-from-client-c': 'hello from client',
+          },
+        }
+      );
+    },
+    { skipDeploy: true }
+  )
+);

packages/cli/test/unit/util/dev/headers.test.ts

@@ -0,0 +1,77 @@
+import { Headers } from 'node-fetch';
+import { applyOverriddenHeaders } from '../../../../src/util/dev/headers';
+
+describe('applyOverriddenHeaders', () => {
+  it('do nothing if x-middleware-override-headers is not set', async () => {
+    const reqHeaders = { a: '1' };
+    const respHeaders = new Headers();
+
+    applyOverriddenHeaders(reqHeaders, respHeaders);
+    expect(reqHeaders).toStrictEqual({ a: '1' });
+  });
+
+  it('adds a new header', async () => {
+    const reqHeaders = { a: '1' };
+    const respHeaders = new Headers({
+      // Define a new header 'b' and keep the existing header 'a'
+      'x-middleware-override-headers': 'a,b',
+      'x-middleware-request-a': '1',
+      'x-middleware-request-b': '2',
+    });
+
+    applyOverriddenHeaders(reqHeaders, respHeaders);
+    expect(reqHeaders).toStrictEqual({ a: '1', b: '2' });
+  });
+
+  it('delete the header if x-middleware-request-* is undefined', async () => {
+    const reqHeaders = { a: '1', b: '2' };
+    const respHeaders = new Headers({
+      // Deletes a new header 'c' and keep the existing headers `a` and `b`
+      'x-middleware-override-headers': 'a,b,c',
+      'x-middleware-request-a': '1',
+      'x-middleware-request-b': '2',
+    });
+
+    applyOverriddenHeaders(reqHeaders, respHeaders);
+    expect(reqHeaders).toStrictEqual({ a: '1', b: '2' });
+  });
+
+  it('updates an existing header', async () => {
+    const reqHeaders = { a: '1', b: '2' };
+    const respHeaders = new Headers({
+      // Modifies the header 'b' and keep the existing header 'a'
+      'x-middleware-override-headers': 'a,b',
+      'x-middleware-request-a': '1',
+      'x-middleware-request-b': 'modified',
+    });
+
+    applyOverriddenHeaders(reqHeaders, respHeaders);
+    expect(reqHeaders).toStrictEqual({ a: '1', b: 'modified' });
+  });
+
+  it('ignores headers listed in NONOVERRIDABLE_HEADERS', async () => {
+    const reqHeaders = { a: '1', host: 'example.com' };
+    const respHeaders = new Headers({
+      // Define a new header 'b' and 'content-length'
+      'x-middleware-override-headers': 'a,b,content-length',
+      'x-middleware-request-a': '1',
+      'x-middleware-request-b': '2',
+      'x-middleware-request-content-length': '128',
+    });
+
+    applyOverriddenHeaders(reqHeaders, respHeaders);
+    expect(reqHeaders).toStrictEqual({ a: '1', b: '2', host: 'example.com' });
+  });
+
+  it('deletes an existing header', async () => {
+    const reqHeaders = { a: '1', b: '2' };
+    const respHeaders = new Headers({
+      // Deletes the header 'a' and keep the existing header 'b'
+      'x-middleware-override-headers': 'b',
+      'x-middleware-request-b': '2',
+    });
+
+    applyOverriddenHeaders(reqHeaders, respHeaders);
+    expect(reqHeaders).toStrictEqual({ b: '2' });
+  });
+});