Support overriding request headers in middlewares (#41380)

This PR adds a feature in middleware to add, modify, or delete request headers. This feature is quite useful to pass data from middleware to Serverless/Edge API routes.

### Questions for Reviewers

- Should we deny modifying standard request headers like `Transfer-Encoding`?
- Should we throw an error if the header is too large? Real-world HTTP servers will accept up to only 8KB - 32KB.

### New APIs

Adds a new option `request.headers` to the `MiddlewareResponseInit` parameter in `NextResponse.next()` and `NextResponse.rewrite()`. It's a [`Header`](https://developer.mozilla.org/en-US/docs/Web/API/Headers) object holding *all* request headers. Specifically:

```ts
interface MiddlewareResponseInit extends ResponseInit {
  request?: {
    headers?: Headers
  }
}
```

### Example

```ts
// pages/api/hello.ts
export default (req, res) => {
  const valueFromMiddleware = req.headers['x-hello-from-middleware']
  return res.send(valueFromMiddleware)
}

// middleware.ts
import { NextRequest, NextResponse } from 'next/server'

export default function middleware(request: NextRequest) {
  // Clone request headers
  const headers = new Headers(request.headers);
  // Add a new request header
  headers.set('x-hello-from-middleware', 'foo');
  // Delete a request header from the client
  headers.delete('x-from-client');

  const resp = NextResponse.next({
    // New option `request.headers` which accepts a Headers object
    // overrides request headers with the specified new ones.
    request: {
      headers
    }
  });

  // You can still set *response* headers to the client, as before.
  resp.headers.set('x-hello-client', 'bar');
  return resp;
}
```

### New middleware headers

- `x-middleware-override-headers`: A comma separated list of *all* request header names. Headers not listed will be deleted.
- `x-middleware-request-<name>`: A new value for the header `<name>`.

## Related Discussions

- #31188
- #39300

## Bug

- [ ] Related issues linked using `fixes #number`
- [ ] Integration tests added
- [ ] Errors have a helpful link attached, see `contributing.md`

## Feature

- [x] Implements an existing feature request or RFC. Make sure the feature request has been accepted for implementation before opening a PR.
- [ ] Related issues linked using `fixes #number`
- [x] Integration tests added
- [x] Documentation added
- [ ] Telemetry added. In case of a feature if it's used or not.
- [ ] Errors have a helpful link attached, see `contributing.md`

## Documentation / Examples

- [ ] Make sure the linting passes by running `pnpm lint`
- [ ] The "examples guidelines" are followed from [our contributing doc](https://github.com/vercel/next.js/blob/canary/contributing/examples/adding-examples.md)

packages/next/server/next-server.ts

@@ -1882,6 +1882,37 @@ export default class NextNodeServer extends BaseServer {
               result.response.headers.set('x-middleware-rewrite', rel)
             }
 
+            if (result.response.headers.has('x-middleware-override-headers')) {
+              const overriddenHeaders: Set<string> = new Set()
+              for (const key of result.response.headers
+                .get('x-middleware-override-headers')!
+                .split(',')) {
+                overriddenHeaders.add(key.trim())
+              }
+
+              result.response.headers.delete('x-middleware-override-headers')
+
+              // Delete headers.
+              for (const key of Object.keys(req.headers)) {
+                if (!overriddenHeaders.has(key)) {
+                  delete req.headers[key]
+                }
+              }
+
+              // Update or add headers.
+              for (const key of overriddenHeaders.keys()) {
+                const valueKey = 'x-middleware-request-' + key
+                const newValue = result.response.headers.get(valueKey)
+                const oldValue = req.headers[key]
+
+                if (oldValue !== newValue) {
+                  req.headers[key] = newValue === null ? undefined : newValue
+                }
+
+                result.response.headers.delete(valueKey)
+              }
+            }
+
             if (result.response.headers.has('Location')) {
               const value = result.response.headers.get('Location')!
               const rel = relativizeURL(value, initUrl)

packages/next/server/web/spec-extension/response.ts

@@ -7,6 +7,25 @@ import { NextCookies } from './cookies'
 const INTERNALS = Symbol('internal response')
 const REDIRECTS = new Set([301, 302, 303, 307, 308])
 
+function handleMiddlewareField(
+  init: MiddlewareResponseInit | undefined,
+  headers: Headers
+) {
+  if (init?.request?.headers) {
+    if (!(init.request.headers instanceof Headers)) {
+      throw new Error('request.headers must be an instance of Headers')
+    }
+
+    const keys = []
+    for (const [key, value] of init.request.headers) {
+      headers.set('x-middleware-request-' + key, value)
+      keys.push(key)
+    }
+
+    headers.set('x-middleware-override-headers', keys.join(','))
+  }
+}
+
 export class NextResponse extends Response {
   [INTERNALS]: {
     cookies: NextCookies
@@ -71,15 +90,22 @@ export class NextResponse extends Response {
     })
   }
 
-  static rewrite(destination: string | NextURL | URL, init?: ResponseInit) {
+  static rewrite(
+    destination: string | NextURL | URL,
+    init?: MiddlewareResponseInit
+  ) {
     const headers = new Headers(init?.headers)
     headers.set('x-middleware-rewrite', validateURL(destination))
+
+    handleMiddlewareField(init, headers)
     return new NextResponse(null, { ...init, headers })
   }
 
-  static next(init?: ResponseInit) {
+  static next(init?: MiddlewareResponseInit) {
     const headers = new Headers(init?.headers)
     headers.set('x-middleware-next', '1')
+
+    handleMiddlewareField(init, headers)
     return new NextResponse(null, { ...init, headers })
   }
 }
@@ -92,3 +118,17 @@ interface ResponseInit extends globalThis.ResponseInit {
   }
   url?: string
 }
+
+interface ModifiedRequest {
+  /**
+   * If this is set, the request headers will be overridden with this value.
+   */
+  headers?: Headers
+}
+
+interface MiddlewareResponseInit extends globalThis.ResponseInit {
+  /**
+   * These fields will override the request from clients.
+   */
+  request?: ModifiedRequest
+}

test/e2e/app-dir/app-middleware.test.ts

@@ -0,0 +1,129 @@
+/* eslint-env jest */
+
+import { NextInstance } from 'test/lib/next-modes/base'
+import { fetchViaHTTP } from 'next-test-utils'
+import { createNext, FileRef } from 'e2e-utils'
+import cheerio from 'cheerio'
+import path from 'path'
+
+describe('app-dir with middleware', () => {
+  if ((global as any).isNextDeploy) {
+    it('should skip next deploy for now', () => {})
+    return
+  }
+
+  if (process.env.NEXT_TEST_REACT_VERSION === '^17') {
+    it('should skip for react v17', () => {})
+    return
+  }
+
+  let next: NextInstance
+
+  afterAll(() => next.destroy())
+  beforeAll(async () => {
+    next = await createNext({
+      files: new FileRef(path.join(__dirname, 'app-middleware')),
+      dependencies: {
+        react: 'experimental',
+        'react-dom': 'experimental',
+      },
+    })
+  })
+
+  describe.each([
+    {
+      title: 'Serverless Functions',
+      path: '/api/dump-headers-serverless',
+      toJson: (res: Response) => res.json(),
+    },
+    {
+      title: 'Edge Functions',
+      path: '/api/dump-headers-edge',
+      toJson: (res: Response) => res.json(),
+    },
+    {
+      title: 'next/headers',
+      path: '/headers',
+      toJson: async (res: Response) => {
+        const $ = cheerio.load(await res.text())
+        return JSON.parse($('#headers').text())
+      },
+    },
+  ])('Mutate request headers for $title', ({ path, toJson }) => {
+    it(`Adds new headers`, async () => {
+      const res = await fetchViaHTTP(next.url, path, null, {
+        headers: {
+          'x-from-client': 'hello-from-client',
+        },
+      })
+      expect(await toJson(res)).toMatchObject({
+        'x-from-client': 'hello-from-client',
+        'x-from-middleware': 'hello-from-middleware',
+      })
+    })
+
+    it(`Deletes headers`, async () => {
+      const res = await fetchViaHTTP(
+        next.url,
+        path,
+        {
+          'remove-headers': 'x-from-client1,x-from-client2',
+        },
+        {
+          headers: {
+            'x-from-client1': 'hello-from-client',
+            'X-From-Client2': 'hello-from-client',
+          },
+        }
+      )
+
+      const json = await toJson(res)
+      expect(json).not.toHaveProperty('x-from-client1')
+      expect(json).not.toHaveProperty('X-From-Client2')
+      expect(json).toMatchObject({
+        'x-from-middleware': 'hello-from-middleware',
+      })
+
+      // Should not be included in response headers.
+      expect(res.headers.get('x-middleware-override-headers')).toBeNull()
+      expect(
+        res.headers.get('x-middleware-request-x-from-middleware')
+      ).toBeNull()
+      expect(res.headers.get('x-middleware-request-x-from-client1')).toBeNull()
+      expect(res.headers.get('x-middleware-request-x-from-client2')).toBeNull()
+    })
+
+    it(`Updates headers`, async () => {
+      const res = await fetchViaHTTP(
+        next.url,
+        path,
+        {
+          'update-headers':
+            'x-from-client1=new-value1,x-from-client2=new-value2',
+        },
+        {
+          headers: {
+            'x-from-client1': 'old-value1',
+            'X-From-Client2': 'old-value2',
+            'x-from-client3': 'old-value3',
+          },
+        }
+      )
+      expect(await toJson(res)).toMatchObject({
+        'x-from-client1': 'new-value1',
+        'x-from-client2': 'new-value2',
+        'x-from-client3': 'old-value3',
+        'x-from-middleware': 'hello-from-middleware',
+      })
+
+      // Should not be included in response headers.
+      expect(res.headers.get('x-middleware-override-headers')).toBeNull()
+      expect(
+        res.headers.get('x-middleware-request-x-from-middleware')
+      ).toBeNull()
+      expect(res.headers.get('x-middleware-request-x-from-client1')).toBeNull()
+      expect(res.headers.get('x-middleware-request-x-from-client2')).toBeNull()
+      expect(res.headers.get('x-middleware-request-x-from-client3')).toBeNull()
+    })
+  })
+})

test/e2e/app-dir/app-middleware/app/headers/page.js

@@ -0,0 +1,10 @@
+import { headers } from 'next/headers'
+
+export default function SSRPage() {
+  const headersObj = Object.fromEntries(headers())
+  return (
+    <>
+      <p id="headers">{JSON.stringify(headersObj)}</p>
+    </>
+  )
+}

test/e2e/app-dir/app-middleware/app/layout.js

@@ -0,0 +1,10 @@
+export default function Layout({ children }) {
+  return (
+    <html lang="en">
+      <head>
+        <title>app-middleware</title>
+      </head>
+      <body>{children}</body>
+    </html>
+  )
+}

test/e2e/app-dir/app-middleware/middleware.js

@@ -0,0 +1,30 @@
+import { NextResponse } from 'next/server'
+
+/**
+ * @param {import('next/server').NextRequest} request
+ */
+export async function middleware(request) {
+  const headers = new Headers(request.headers)
+  headers.set('x-from-middleware', 'hello-from-middleware')
+
+  const removeHeaders = request.nextUrl.searchParams.get('remove-headers')
+  if (removeHeaders) {
+    for (const key of removeHeaders.split(',')) {
+      headers.delete(key)
+    }
+  }
+
+  const updateHeader = request.nextUrl.searchParams.get('update-headers')
+  if (updateHeader) {
+    for (const kv of updateHeader.split(',')) {
+      const [key, value] = kv.split('=')
+      headers.set(key, value)
+    }
+  }
+
+  return NextResponse.next({
+    request: {
+      headers,
+    },
+  })
+}

test/e2e/app-dir/app-middleware/next.config.js

@@ -0,0 +1,7 @@
+module.exports = {
+  experimental: {
+    appDir: true,
+    legacyBrowsers: false,
+    browsersListForSwc: true,
+  },
+}

test/e2e/app-dir/app-middleware/pages/api/dump-headers-edge.js

@@ -0,0 +1,11 @@
+export const config = {
+  runtime: 'experimental-edge',
+}
+
+export default (req) => {
+  return Response.json(Object.fromEntries(req.headers.entries()), {
+    headers: {
+      'headers-from-edge-function': '1',
+    },
+  })
+}

test/e2e/app-dir/app-middleware/pages/api/dump-headers-serverless.js

@@ -0,0 +1,6 @@
+export default (req, res) => {
+  return res
+    .status(200)
+    .setHeader('headers-from-serverless', '1')
+    .json(req.headers)
+}

test/e2e/middleware-request-header-overrides/app/.gitignore

@@ -0,0 +1 @@
+.vercel

test/e2e/middleware-request-header-overrides/app/middleware.js

@@ -0,0 +1,30 @@
+import { NextResponse } from 'next/server'
+
+/**
+ * @param {import('next/server').NextRequest} request
+ */
+export async function middleware(request) {
+  const headers = new Headers(request.headers)
+  headers.set('x-from-middleware', 'hello-from-middleware')
+
+  const removeHeaders = request.nextUrl.searchParams.get('remove-headers')
+  if (removeHeaders) {
+    for (const key of removeHeaders.split(',')) {
+      headers.delete(key)
+    }
+  }
+
+  const updateHeader = request.nextUrl.searchParams.get('update-headers')
+  if (updateHeader) {
+    for (const kv of updateHeader.split(',')) {
+      const [key, value] = kv.split('=')
+      headers.set(key, value)
+    }
+  }
+
+  return NextResponse.next({
+    request: {
+      headers,
+    },
+  })
+}

test/e2e/middleware-request-header-overrides/app/next.config.js

@@ -0,0 +1 @@
+module.exports = {}

test/e2e/middleware-request-header-overrides/app/pages/api/dump-headers-edge.js

@@ -0,0 +1,11 @@
+export const config = {
+  runtime: 'experimental-edge',
+}
+
+export default (req) => {
+  return Response.json(Object.fromEntries(req.headers.entries()), {
+    headers: {
+      'headers-from-edge-function': '1',
+    },
+  })
+}

test/e2e/middleware-request-header-overrides/app/pages/api/dump-headers-serverless.js

@@ -0,0 +1,6 @@
+export default (req, res) => {
+  return res
+    .status(200)
+    .setHeader('headers-from-serverless', '1')
+    .json(req.headers)
+}

test/e2e/middleware-request-header-overrides/app/pages/ssr-page.js

@@ -0,0 +1,15 @@
+export default function SSRPage({ headers }) {
+  return (
+    <>
+      <p id="headers">{JSON.stringify(headers)}</p>
+    </>
+  )
+}
+
+export const getServerSideProps = (ctx) => {
+  return {
+    props: {
+      headers: ctx.req.headers,
+    },
+  }
+}