Generate static html for bots (#35004)

* Generate static html for bots

* fix lint

* refactor error handling against comment

* fix streaming on edge for bots

* inline doResolve

packages/next/server/base-server.ts

@@ -980,12 +980,12 @@ export default abstract class Server {
       query: NextParsedUrlQuery
     }
   ): Promise<void> {
-    const userAgent = partialContext.req.headers['user-agent']
+    const isBotRequest = isBot(partialContext.req.headers['user-agent'] || '')
     const ctx = {
       ...partialContext,
       renderOpts: {
         ...this.renderOpts,
-        supportsDynamicHTML: userAgent ? !isBot(userAgent) : false,
+        supportsDynamicHTML: !isBotRequest,
       },
     } as const
     const payload = await fn(ctx)
@@ -1175,11 +1175,13 @@ export default abstract class Server {
     }
 
     if (opts.supportsDynamicHTML === true) {
+      const isBotRequest = isBot(req.headers['user-agent'] || '')
       // Disable dynamic HTML in cases that we know it won't be generated,
       // so that we can continue generating a cache key when possible.
       opts.supportsDynamicHTML =
         !isSSG &&
         !isLikeServerless &&
+        !isBotRequest &&
         !query.amp &&
         typeof components.Document?.getInitialProps !== 'function'
     }

packages/next/server/render.tsx

@@ -67,6 +67,7 @@ import isError from '../lib/is-error'
 import { readableStreamTee } from './web/utils'
 import { ImageConfigContext } from '../shared/lib/image-config-context'
 import { FlushEffectsContext } from '../shared/lib/flush-effects'
+import { execOnce } from '../shared/lib/utils'
 
 let optimizeAmp: typeof import('./optimize-amp').default
 let getFontDefinitionFromManifest: typeof import('./font-utils').getFontDefinitionFromManifest
@@ -1349,6 +1350,7 @@ export async function renderToHTML(
                   ))}
                 </>
               ),
+              generateStaticHTML: true,
             })
 
             const flushed = await streamToString(flushEffectStream)
@@ -1360,6 +1362,7 @@ export async function renderToHTML(
             element: content,
             suffix,
             dataStream: serverComponentsInlinedTransformStream?.readable,
+            generateStaticHTML: generateStaticHTML || !hasConcurrentFeatures,
             flushEffectHandler,
           })
         }
@@ -1492,6 +1495,7 @@ export async function renderToHTML(
     const documentStream = await renderToStream({
       ReactDOMServer,
       element: document,
+      generateStaticHTML: true,
     })
     documentHTML = await streamToString(documentStream)
   } else {
@@ -1741,67 +1745,62 @@ function createFlushEffectStream(
   })
 }
 
-function renderToStream({
+async function renderToStream({
   ReactDOMServer,
   element,
   suffix,
   dataStream,
+  generateStaticHTML,
   flushEffectHandler,
 }: {
   ReactDOMServer: typeof import('react-dom/server')
   element: React.ReactElement
   suffix?: string
   dataStream?: ReadableStream<Uint8Array>
+  generateStaticHTML: boolean
   flushEffectHandler?: () => Promise<string>
 }): Promise<ReadableStream<Uint8Array>> {
-  return new Promise(async (resolve, reject) => {
-    let resolved = false
-
-    const closeTag = '</body></html>'
-    const suffixUnclosed = suffix ? suffix.split(closeTag)[0] : null
-
-    const doResolve = (renderStream: ReadableStream<Uint8Array>) => {
-      if (!resolved) {
-        resolved = true
-
-        // React will call our callbacks synchronously, so we need to
-        // defer to a microtask to ensure `stream` is set.
-        resolve(
-          Promise.resolve().then(() => {
-            const transforms: Array<TransformStream<Uint8Array, Uint8Array>> = [
-              createBufferedTransformStream(),
-              flushEffectHandler
-                ? createFlushEffectStream(flushEffectHandler)
-                : null,
-              suffixUnclosed != null
-                ? createPrefixStream(suffixUnclosed)
-                : null,
-              dataStream ? createInlineDataStream(dataStream) : null,
-              suffixUnclosed != null ? createSuffixStream(closeTag) : null,
-            ].filter(Boolean) as any
-
-            return transforms.reduce(
-              (readable, transform) => pipeThrough(readable, transform),
-              renderStream
-            )
-          })
-        )
+  const closeTag = '</body></html>'
+  const suffixUnclosed = suffix ? suffix.split(closeTag)[0] : null
+
+  let completeCallback: (value?: unknown) => void
+  const allComplete = new Promise((resolveError, rejectError) => {
+    completeCallback = execOnce((err: unknown) => {
+      if (err) {
+        rejectError(err)
+      } else {
+        resolveError(null)
       }
-    }
-
-    const renderStream: ReadableStream<Uint8Array> = await (
-      ReactDOMServer as any
-    ).renderToReadableStream(element, {
-      onError(err: Error) {
-        if (!resolved) {
-          resolved = true
-          reject(err)
-        }
-      },
     })
+  })
 
-    doResolve(renderStream)
+  const renderStream: ReadableStream<Uint8Array> = await (
+    ReactDOMServer as any
+  ).renderToReadableStream(element, {
+    onError(err: Error) {
+      completeCallback(err)
+    },
+    onCompleteAll() {
+      completeCallback()
+    },
   })
+
+  if (generateStaticHTML) {
+    await allComplete
+  }
+
+  const transforms: Array<TransformStream<Uint8Array, Uint8Array>> = [
+    createBufferedTransformStream(),
+    flushEffectHandler ? createFlushEffectStream(flushEffectHandler) : null,
+    suffixUnclosed != null ? createPrefixStream(suffixUnclosed) : null,
+    dataStream ? createInlineDataStream(dataStream) : null,
+    suffixUnclosed != null ? createSuffixStream(closeTag) : null,
+  ].filter(Boolean) as any
+
+  return transforms.reduce(
+    (readable, transform) => pipeThrough(readable, transform),
+    renderStream
+  )
 }
 
 function encodeText(input: string) {

packages/next/server/web-server.ts

@@ -131,7 +131,7 @@ export default class NextWebServer extends BaseServer {
       query,
       {
         ...renderOpts,
-        supportsDynamicHTML: true,
+        // supportsDynamicHTML: true,
         disableOptimizedLoading: true,
         runtime: 'edge',
       }
@@ -175,7 +175,9 @@ export default class NextWebServer extends BaseServer {
         // Not implemented: on/removeListener
       } as any)
     } else {
-      res.body(await options.result.toUnchunkedString())
+      // TODO: generate Etag
+      const payload = await options.result.toUnchunkedString()
+      res.body(payload)
     }
 
     res.send()

test/integration/react-18/test/index.test.js

@@ -10,7 +10,6 @@ import {
   nextBuild,
   nextStart,
   renderViaHTTP,
-  fetchViaHTTP,
   hasRedbox,
   getRedboxHeader,
 } from 'next-test-utils'
@@ -145,40 +144,6 @@ function runTestsAgainstRuntime(runtime) {
           )
         })
       }
-
-      it('should stream to users', async () => {
-        const res = await fetchViaHTTP(context.appPort, '/ssr')
-        expect(res.headers.get('etag')).toBeNull()
-      })
-
-      it('should not stream to bots', async () => {
-        const res = await fetchViaHTTP(
-          context.appPort,
-          '/ssr',
-          {},
-          {
-            headers: {
-              'user-agent': 'Googlebot',
-            },
-          }
-        )
-        expect(res.headers.get('etag')).toBeDefined()
-      })
-
-      it('should not stream to google pagerender bot', async () => {
-        const res = await fetchViaHTTP(
-          context.appPort,
-          '/ssr',
-          {},
-          {
-            headers: {
-              'user-agent':
-                'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.87 Safari/537.36 Google-PageRenderer Google (+https://developers.google.com/+/web/snippet/)',
-            },
-          }
-        )
-        expect(res.headers.get('etag')).toBeDefined()
-      })
     },
     {
       beforeAll: (env) => {

test/integration/react-streaming-and-server-components/app/components/client-exports.client.js

@@ -0,0 +1,5 @@
+export function Named() {
+  return 'named.client'
+}
+
+export default () => 'default-export-arrow.client'

test/integration/react-streaming-and-server-components/app/pages/err/suspense.js

@@ -4,7 +4,7 @@ let did = false
 function Error() {
   if (!did && typeof window === 'undefined') {
     did = true
-    throw new Error('broken page')
+    throw new Error('oops')
   }
 }
 

test/integration/react-streaming-and-server-components/app/pages/index.server.js

@@ -1,6 +1,4 @@
 import Foo from '../components/foo.client'
-import { Named } from '../components/named.client'
-
 import Link from 'next/link'
 
 const envVar = process.env.ENV_VAR_TEST
@@ -13,9 +11,6 @@ export default function Index({ header, router }) {
       <div>{'path:' + router.pathname}</div>
       <div>{'env:' + envVar}</div>
       <div>{'header:' + header}</div>
-      <div>
-        <Named />
-      </div>
       <div>
         <Foo />
       </div>

test/integration/react-streaming-and-server-components/app/pages/various-exports.server.js

@@ -0,0 +1,26 @@
+// shared named exports
+import { a, b, c, d, e } from '../components/shared-exports'
+// client default, named exports
+import DefaultArrow, {
+  Named as ClientNamed,
+} from '../components/client-exports.client'
+
+export default function Page() {
+  return (
+    <div>
+      <div>
+        {a}
+        {b}
+        {c}
+        {d}
+        {e[0]}
+      </div>
+      <div>
+        <DefaultArrow />
+      </div>
+      <div>
+        <ClientNamed />
+      </div>
+    </div>
+  )
+}

test/integration/react-streaming-and-server-components/test/basic.js

@@ -37,13 +37,12 @@ export default async function basic(context, { env }) {
   })
 
   it('should render 500 error correctly', async () => {
-    const path500HTML = await renderViaHTTP(context.appPort, '/err')
-
+    const html = await renderViaHTTP(context.appPort, '/err')
     if (env === 'dev') {
       // In dev mode it should show the error popup.
-      expect(path500HTML).toContain('Error: oops')
+      expect(html).toContain('Error: oops')
     } else {
-      expect(path500HTML).toContain('custom-500-page')
+      expect(html).toContain('custom-500-page')
     }
   })
 }