Generate static html for bots

packages/next/server/base-server.ts

@@ -985,7 +985,9 @@ export default abstract class Server {
       ...partialContext,
       renderOpts: {
         ...this.renderOpts,
-        supportsDynamicHTML: userAgent ? !isBot(userAgent) : false,
+        supportsDynamicHTML: userAgent
+          ? !isBot(userAgent)
+          : !!this.renderOpts.supportsDynamicHTML,
       },
     } as const
     const payload = await fn(ctx)

packages/next/server/render.tsx

@@ -1349,6 +1349,7 @@ export async function renderToHTML(
                   ))}
                 </>
               ),
+              generateStaticHTML: true,
             })
 
             const flushed = await streamToString(flushEffectStream)
@@ -1360,6 +1361,7 @@ export async function renderToHTML(
             element: content,
             suffix,
             dataStream: serverComponentsInlinedTransformStream?.readable,
+            generateStaticHTML: generateStaticHTML || !hasConcurrentFeatures,
             flushEffectHandler,
           })
         }
@@ -1492,6 +1494,7 @@ export async function renderToHTML(
     const documentStream = await renderToStream({
       ReactDOMServer,
       element: document,
+      generateStaticHTML: true,
     })
     documentHTML = await streamToString(documentStream)
   } else {
@@ -1746,12 +1749,14 @@ function renderToStream({
   element,
   suffix,
   dataStream,
+  generateStaticHTML,
   flushEffectHandler,
 }: {
   ReactDOMServer: typeof import('react-dom/server')
   element: React.ReactElement
   suffix?: string
   dataStream?: ReadableStream<Uint8Array>
+  generateStaticHTML: boolean
   flushEffectHandler?: () => Promise<string>
 }): Promise<ReadableStream<Uint8Array>> {
   return new Promise(async (resolve, reject) => {
@@ -1789,6 +1794,10 @@ function renderToStream({
       }
     }
 
+    let resolveAllComplete: (value?: unknown) => void
+    const allCompleted = new Promise((r) => {
+      resolveAllComplete = r
+    })
     const renderStream: ReadableStream<Uint8Array> = await (
       ReactDOMServer as any
     ).renderToReadableStream(element, {
@@ -1798,8 +1807,14 @@ function renderToStream({
           reject(err)
         }
       },
+      onCompleteAll() {
+        resolveAllComplete()
+      },
     })
 
+    if (generateStaticHTML) {
+      await allCompleted
+    }
     doResolve(renderStream)
   })
 }

packages/next/server/web-server.ts

@@ -3,9 +3,9 @@ import type { RenderOpts } from './render'
 import type RenderResult from './render-result'
 import type { NextParsedUrlQuery } from './request-meta'
 import type { Params } from './router'
-import type { PayloadOptions } from './send-payload'
 import type { LoadComponentsReturnType } from './load-components'
 
+import { PayloadOptions } from './send-payload'
 import BaseServer, { Options } from './base-server'
 import { renderToHTML } from './render'
 
@@ -131,7 +131,6 @@ export default class NextWebServer extends BaseServer {
       query,
       {
         ...renderOpts,
-        supportsDynamicHTML: true,
         disableOptimizedLoading: true,
         runtime: 'edge',
       }
@@ -175,7 +174,9 @@ export default class NextWebServer extends BaseServer {
         // Not implemented: on/removeListener
       } as any)
     } else {
-      res.body(await options.result.toUnchunkedString())
+      // TODO: generate Etag
+      const payload = await options.result.toUnchunkedString()
+      res.body(payload)
     }
 
     res.send()

test/integration/react-18/test/index.test.js

@@ -10,7 +10,6 @@ import {
   nextBuild,
   nextStart,
   renderViaHTTP,
-  fetchViaHTTP,
   hasRedbox,
   getRedboxHeader,
 } from 'next-test-utils'
@@ -145,40 +144,6 @@ function runTestsAgainstRuntime(runtime) {
           )
         })
       }
-
-      it('should stream to users', async () => {
-        const res = await fetchViaHTTP(context.appPort, '/ssr')
-        expect(res.headers.get('etag')).toBeNull()
-      })
-
-      it('should not stream to bots', async () => {
-        const res = await fetchViaHTTP(
-          context.appPort,
-          '/ssr',
-          {},
-          {
-            headers: {
-              'user-agent': 'Googlebot',
-            },
-          }
-        )
-        expect(res.headers.get('etag')).toBeDefined()
-      })
-
-      it('should not stream to google pagerender bot', async () => {
-        const res = await fetchViaHTTP(
-          context.appPort,
-          '/ssr',
-          {},
-          {
-            headers: {
-              'user-agent':
-                'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)',
-            },
-          }
-        )
-        expect(res.headers.get('etag')).toBeDefined()
-      })
     },
     {
       beforeAll: (env) => {

test/integration/react-streaming-and-server-components/app/components/client-exports.client.js

@@ -0,0 +1,5 @@
+export function Named() {
+  return 'named.client'
+}
+
+export default () => 'default-export-arrow.client'

test/integration/react-streaming-and-server-components/app/pages/index.server.js

@@ -1,6 +1,4 @@
 import Foo from '../components/foo.client'
-import { Named } from '../components/named.client'
-
 import Link from 'next/link'
 
 const envVar = process.env.ENV_VAR_TEST
@@ -13,9 +11,6 @@ export default function Index({ header, router }) {
       <div>{'path:' + router.pathname}</div>
       <div>{'env:' + envVar}</div>
       <div>{'header:' + header}</div>
-      <div>
-        <Named />
-      </div>
       <div>
         <Foo />
       </div>

test/integration/react-streaming-and-server-components/app/pages/various-exports.server.js

@@ -0,0 +1,26 @@
+// shared named exports
+import { a, b, c, d, e } from '../components/shared-exports'
+// client default, named exports
+import DefaultArrow, {
+  Named as ClientNamed,
+} from '../components/client-exports.client'
+
+export default function Page() {
+  return (
+    <div>
+      <div>
+        {a}
+        {b}
+        {c}
+        {d}
+        {e[0]}
+      </div>
+      <div>
+        <DefaultArrow />
+      </div>
+      <div>
+        <ClientNamed />
+      </div>
+    </div>
+  )
+}

test/integration/react-streaming-and-server-components/test/index.test.js

@@ -150,9 +150,10 @@ describe('Edge runtime - prod', () => {
     expect(html).toContain('foo.client')
   })
 
-  basic(context, { env: 'prod' })
-  streaming(context, { env: 'prod' })
-  rsc(context, { runtime: 'edge', env: 'prod' })
+  const options = { runtime: 'edge', env: 'prod' }
+  basic(context, options)
+  streaming(context, options)
+  rsc(context, options)
 })
 
 describe('Edge runtime - dev', () => {
@@ -183,16 +184,18 @@ describe('Edge runtime - dev', () => {
     expect(res.headers.get('content-encoding')).toBe('gzip')
   })
 
-  basic(context, { env: 'dev' })
-  streaming(context, { env: 'dev' })
-  rsc(context, { runtime: 'edge', env: 'dev' })
+  const options = { runtime: 'edge', env: 'dev' }
+  basic(context, options)
+  streaming(context, options)
+  rsc(context, options)
 })
 
 const nodejsRuntimeBasicSuite = {
   runTests: (context, env) => {
-    basic(context, { env })
-    streaming(context, { env })
-    rsc(context, { runtime: 'nodejs' })
+    const options = { runtime: 'nodejs', env }
+    basic(context, options)
+    streaming(context, options)
+    rsc(context, options)
 
     if (env === 'prod') {
       it('should generate middleware SSR manifests for Node.js', async () => {

test/integration/react-streaming-and-server-components/test/rsc.js

@@ -28,7 +28,6 @@ export default function (context, { runtime, env }) {
     expect(homeHTML).toContain('header:test-util')
     expect(homeHTML).toContain('path:/')
     expect(homeHTML).toContain('foo.client')
-    expect(homeHTML).toContain('named.client')
   })
 
   it('should reuse the inline flight response without sending extra requests', async () => {
@@ -165,34 +164,20 @@ export default function (context, { runtime, env }) {
     })
   }
 
-  it('should handle multiple named exports correctly', async () => {
-    const clientExportsHTML = await renderViaHTTP(
-      context.appPort,
-      '/client-exports'
-    )
-
-    expect(
-      getNodeBySelector(
-        clientExportsHTML,
-        '#__next > div > #named-exports'
-      ).text()
-    ).toBe('abcde')
-    expect(
-      getNodeBySelector(
-        clientExportsHTML,
-        '#__next > div > #default-exports-arrow'
-      ).text()
-    ).toBe('client-default-export-arrow')
-
-    const browser = await webdriver(context.appPort, '/client-exports')
-    const textNamedExports = await browser
-      .waitForElementByCss('#named-exports')
-      .text()
-    const textDefaultExportsArrow = await browser
-      .waitForElementByCss('#default-exports-arrow')
-      .text()
-    expect(textNamedExports).toBe('abcde')
-    expect(textDefaultExportsArrow).toBe('client-default-export-arrow')
+  it('should handle various kinds of exports correctly', async () => {
+    const html = await renderViaHTTP(context.appPort, '/various-exports')
+    const content = getNodeBySelector(html, '#__next').text()
+
+    expect(content).toContain('abcde')
+    expect(content).toContain('default-export-arrow.client')
+    expect(content).toContain('named.client')
+
+    const browser = await webdriver(context.appPort, '/various-exports')
+    const hydratedContent = await browser.waitForElementByCss('#__next').text()
+
+    expect(hydratedContent).toContain('abcde')
+    expect(hydratedContent).toContain('default-export-arrow.client')
+    expect(hydratedContent).toContain('named.client')
   })
 
   it('should handle 404 requests and missing routes correctly', async () => {