Fix app static generation cases

packages/next/export/worker.ts

@@ -96,6 +96,7 @@ interface RenderOpts {
   defaultLocale?: string
   domainLocales?: DomainLocale[]
   trailingSlash?: boolean
+  supportsDynamicHTML?: boolean
 }
 
 type ComponentModule = ComponentType<{}> & {
@@ -389,6 +390,7 @@ export default async function exportPage({
             ? requireFontManifest(distDir, serverless)
             : null,
           locale: locale as string,
+          supportsDynamicHTML: false,
         }
 
         // during build we attempt rendering app dir paths
@@ -406,8 +408,7 @@ export default async function exportPage({
               res as any,
               page,
               query,
-              curRenderOpts as any,
-              true
+              curRenderOpts as any
             )
             const html = result?.toUnchunkedString()
             const flightData = (curRenderOpts as any).pageData

packages/next/server/app-render.tsx

@@ -33,6 +33,7 @@ import { REDIRECT_ERROR_CODE } from '../client/components/redirect'
 import { NextCookies } from './web/spec-extension/cookies'
 import { DYNAMIC_ERROR_CODE } from '../client/components/hooks-server-context'
 import { NOT_FOUND_ERROR_CODE } from '../client/components/not-found'
+import { Writable } from 'stream'
 
 const INTERNAL_HEADERS_INSTANCE = Symbol('internal for headers readonly')
 
@@ -658,9 +659,22 @@ export async function renderToHTMLOrFlight(
   res: ServerResponse,
   pathname: string,
   query: NextParsedUrlQuery,
-  renderOpts: RenderOpts,
-  isStaticGeneration: boolean = false
+  renderOpts: RenderOpts
 ): Promise<RenderResult | null> {
+  /**
+   * Rules of Static & Dynamic HTML:
+   *
+   *    1.) We must generate static HTML unless the caller explicitly opts
+   *        in to dynamic HTML support.
+   *
+   *    2.) If dynamic HTML support is requested, we must honor that request
+   *        or throw an error. It is the sole responsibility of the caller to
+   *        ensure they aren't e.g. requesting dynamic HTML for an AMP page.
+   *
+   * These rules help ensure that other existing features like request caching,
+   * coalescing, and ISR continue working as intended.
+   */
+  const isStaticGeneration = renderOpts.supportsDynamicHTML !== true
   const isFlight = req.headers.__rsc__ !== undefined
 
   const capturedErrors: Error[] = []
@@ -683,7 +697,6 @@ export async function renderToHTMLOrFlight(
     subresourceIntegrityManifest,
     serverComponentManifest,
     serverCSSManifest = {},
-    supportsDynamicHTML,
     ComponentMod,
     dev,
   } = renderOpts
@@ -721,12 +734,18 @@ export async function renderToHTMLOrFlight(
     /**
      * Router state provided from the client-side router. Used to handle rendering from the common layout down.
      */
-    const providedFlightRouterState: FlightRouterState = isFlight
+    let providedFlightRouterState: FlightRouterState = isFlight
       ? req.headers.__next_router_state_tree__
         ? JSON.parse(req.headers.__next_router_state_tree__ as string)
-        : {}
+        : []
       : undefined
 
+    if (isStaticGeneration) {
+      // TODO-APP: filter on the client for static instead?
+      // currently we fail to detect new root layouts
+      providedFlightRouterState = ['', { children: ['', {}] }]
+    }
+
     /**
      * The tree created in next-app-loader that holds component segments and modules
      */
@@ -1104,23 +1123,22 @@ export async function renderToHTMLOrFlight(
       }
     }
 
-    /**
-     * Rules of Static & Dynamic HTML:
-     *
-     *    1.) We must generate static HTML unless the caller explicitly opts
-     *        in to dynamic HTML support.
-     *
-     *    2.) If dynamic HTML support is requested, we must honor that request
-     *        or throw an error. It is the sole responsibility of the caller to
-     *        ensure they aren't e.g. requesting dynamic HTML for an AMP page.
-     *
-     * These rules help ensure that other existing features like request caching,
-     * coalescing, and ISR continue working as intended.
-     */
-    const generateStaticHTML = supportsDynamicHTML !== true
+    const streamToBufferedResult = async (
+      renderResult: RenderResult
+    ): Promise<string> => {
+      const renderChunks: Buffer[] = []
+      const writable = new Writable({
+        write(chunk, _encoding, callback) {
+          renderChunks.push(chunk)
+          callback()
+        },
+      })
+      await renderResult.pipe(writable)
+      return Buffer.concat(renderChunks).toString()
+    }
 
     // Handle Flight render request. This is only used when client-side navigating. E.g. when you `router.push('/dashboard')` or `router.reload()`.
-    if (isFlight) {
+    const generateFlight = async (): Promise<RenderResult> => {
       // TODO-APP: throw on invalid flightRouterState
       /**
        * Use router state to decide at what common layout to render the page.
@@ -1252,15 +1270,14 @@ export async function renderToHTMLOrFlight(
         }
       ).pipeThrough(createBufferedTransformStream())
 
-      if (generateStaticHTML) {
-        let staticHtml = Buffer.from(
-          (await readable.getReader().read()).value || ''
-        ).toString()
-        return new FlightRenderResult(staticHtml)
-      }
       return new FlightRenderResult(readable)
     }
 
+    if (isFlight && !isStaticGeneration) {
+      res.setHeader('x-next-dynamic-rsc', '1')
+      return generateFlight()
+    }
+
     // Below this line is handling for rendering to HTML.
 
     // Create full component tree from root to leaf.
@@ -1394,7 +1411,7 @@ export async function renderToHTMLOrFlight(
 
         return await continueFromInitialStream(renderStream, {
           dataStream: serverComponentsInlinedTransformStream?.readable,
-          generateStaticHTML: generateStaticHTML,
+          generateStaticHTML: isStaticGeneration,
           getServerInsertedHTML,
           serverInsertedHTMLToHead: true,
           polyfills,
@@ -1426,39 +1443,42 @@ export async function renderToHTMLOrFlight(
 
         return await continueFromInitialStream(renderStream, {
           dataStream: serverComponentsInlinedTransformStream?.readable,
-          generateStaticHTML: generateStaticHTML,
+          generateStaticHTML: isStaticGeneration,
           getServerInsertedHTML,
           serverInsertedHTMLToHead: true,
           polyfills,
           dev,
         })
       }
     }
+    const renderResult = new RenderResult(await bodyResult())
 
-    if (generateStaticHTML) {
-      const readable = await bodyResult()
-      let staticHtml = Buffer.from(
-        (await readable.getReader().read()).value || ''
-      ).toString()
-
+    if (isStaticGeneration) {
+      const htmlResult = await streamToBufferedResult(renderResult)
       // if we encountered any unexpected errors during build
       // we fail the prerendering phase and the build
       if (capturedErrors.length > 0) {
         throw capturedErrors[0]
       }
+      // const before = Buffer.concat(
+      //   serverComponentsRenderOpts.rscChunks
+      // ).toString()
+
+      // TODO-APP: derive this from same pass to prevent additional
+      // render during static generation
+      const filteredFlightData = await streamToBufferedResult(
+        await generateFlight()
+      )
 
-      ;(renderOpts as any).pageData = Buffer.concat(
-        serverComponentsRenderOpts.rscChunks
-      ).toString()
+      ;(renderOpts as any).pageData = filteredFlightData
       ;(renderOpts as any).revalidate =
         typeof staticGenerationStore?.revalidate === 'undefined'
           ? defaultRevalidate
           : staticGenerationStore?.revalidate
 
-      return new RenderResult(staticHtml)
+      return new RenderResult(htmlResult)
     }
-
-    return new RenderResult(await bodyResult())
+    return renderResult
   }
 
   const initialStaticGenerationStore = {

packages/next/server/base-server.ts

@@ -238,7 +238,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
 
   protected abstract getPublicDir(): string
   protected abstract getHasStaticDir(): boolean
-  protected abstract getHasAppDir(): boolean
+  protected abstract getHasAppDir(dev: boolean): boolean
   protected abstract getPagesManifest(): PagesManifest | undefined
   protected abstract getAppPathsManifest(): PagesManifest | undefined
   protected abstract getBuildId(): string
@@ -355,7 +355,6 @@ export default abstract class Server<ServerOptions extends Options = Options> {
         : require('path').join(this.dir, this.nextConfig.distDir)
     this.publicDir = this.getPublicDir()
     this.hasStaticDir = !minimalMode && this.getHasStaticDir()
-    this.hasAppDir = this.getHasAppDir()
 
     // Only serverRuntimeConfig needs the default
     // publicRuntimeConfig gets it's default in client/index.js
@@ -369,6 +368,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     this.buildId = this.getBuildId()
     this.minimalMode = minimalMode || !!process.env.NEXT_PRIVATE_MINIMAL_MODE
 
+    this.hasAppDir = this.getHasAppDir(dev)
     const serverComponents = this.hasAppDir
     this.serverComponentManifest = serverComponents
       ? this.getServerComponentManifest()
@@ -1001,17 +1001,6 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     const hasGetInitialProps = !!components.Component?.getInitialProps
     let isSSG = !!components.getStaticProps
 
-    // Toggle whether or not this is a Data request
-    const isDataReq =
-      !!(
-        query.__nextDataReq ||
-        (req.headers['x-nextjs-data'] &&
-          (this.serverOptions as any).webServerConfig)
-      ) &&
-      (isSSG || hasServerProps)
-
-    delete query.__nextDataReq
-
     // Compute the iSSG cache key. We use the rewroteUrl since
     // pages with fallback: false are allowed to be rewritten to
     // and we need to look up the path by the rewritten path
@@ -1040,8 +1029,33 @@ export default abstract class Server<ServerOptions extends Options = Options> {
 
       if (hasFallback || staticPaths?.includes(resolvedUrlPathname)) {
         isSSG = true
+      } else if (!this.renderOpts.dev) {
+        const manifest = this.getPrerenderManifest()
+        isSSG =
+          isSSG || !!manifest.routes[pathname === '/index' ? '/' : pathname]
+      }
+    }
+
+    // Toggle whether or not this is a Data request
+    let isDataReq =
+      !!(
+        query.__nextDataReq ||
+        (req.headers['x-nextjs-data'] &&
+          (this.serverOptions as any).webServerConfig)
+      ) &&
+      (isSSG || hasServerProps)
+
+    if (isAppPath && req.headers['__rsc__']) {
+      res.setHeader('content-type', 'application/octet-stream')
+
+      if (isSSG) {
+        isDataReq = true
+        // strip header so we generate HTML still
+        delete req.headers['__rsc__']
+        delete req.headers['__next_router_state_tree__']
       }
     }
+    delete query.__nextDataReq
 
     // normalize req.url for SSG paths as it is not exposed
     // to getStaticProps and the asPath should not expose /_next/data
@@ -1206,7 +1220,7 @@ export default abstract class Server<ServerOptions extends Options = Options> {
     }
 
     let ssgCacheKey =
-      isPreviewMode || !isSSG || opts.supportsDynamicHTML || isFlightRequest
+      isPreviewMode || !isSSG || opts.supportsDynamicHTML
         ? null // Preview mode, manual revalidate, flight request can bypass the cache
         : `${locale ? `/${locale}` : ''}${
             (pathname === '/' || resolvedUrlPathname === '/') && locale
@@ -1566,7 +1580,11 @@ export default abstract class Server<ServerOptions extends Options = Options> {
       return {
         type: isDataReq ? 'json' : 'html',
         body: isDataReq
-          ? RenderResult.fromStatic(JSON.stringify(cachedData.pageData))
+          ? RenderResult.fromStatic(
+              isAppPath
+                ? (cachedData.pageData as string)
+                : JSON.stringify(cachedData.pageData)
+            )
           : cachedData.html,
         revalidateOptions,
       }

packages/next/server/next-server.ts

@@ -474,11 +474,16 @@ export default class NextNodeServer extends BaseServer {
     ]
   }
 
-  protected getHasAppDir(): boolean {
-    const appDirectory = join(this.dir, 'app')
-    return (
-      fs.existsSync(appDirectory) && fs.statSync(appDirectory).isDirectory()
-    )
+  protected getHasAppDir(dev: boolean): boolean {
+    const appDirectory = dev
+      ? join(this.dir, 'app')
+      : join(this.serverDistDir, 'app')
+
+    try {
+      return fs.statSync(appDirectory).isDirectory()
+    } catch (err) {
+      return false
+    }
   }
 
   protected generateStaticRoutes(): Route[] {

packages/next/server/render-result.ts

@@ -1,4 +1,5 @@
 import type { ServerResponse } from 'http'
+import { Writable } from 'stream'
 
 type ContentTypeOption = string | undefined
 
@@ -27,7 +28,7 @@ export default class RenderResult {
     return this._result
   }
 
-  pipe(res: ServerResponse): Promise<void> {
+  pipe(res: ServerResponse | Writable): Promise<void> {
     if (typeof this._result === 'string') {
       throw new Error(
         'invariant: static responses cannot be piped. This is a bug in Next.js'

packages/next/server/web-server.ts

@@ -378,8 +378,7 @@ export default class NextWebServer extends BaseServer<WebServerOptions> {
         Object.assign(renderOpts, {
           disableOptimizedLoading: true,
           runtime: 'experimental-edge',
-        }),
-        !!pagesRenderToHTML
+        })
       )
     } else {
       throw new Error(`Invariant: curRenderToHTML is missing`)