on-demand-entry-handler.ts

import type ws from 'ws'
import origDebug from 'next/dist/compiled/debug'
import type { webpack } from 'next/dist/compiled/webpack/webpack'
import type { NextConfigComplete } from '../config-shared'
import { EventEmitter } from 'events'
import { findPageFile } from '../lib/find-page-file'
import { runDependingOnPageType } from '../../build/entries'
import { join, posix } from 'path'
import { normalizePathSep } from '../../shared/lib/page-path/normalize-path-sep'
import { normalizePagePath } from '../../shared/lib/page-path/normalize-page-path'
import { ensureLeadingSlash } from '../../shared/lib/page-path/ensure-leading-slash'
import { removePagePathTail } from '../../shared/lib/page-path/remove-page-path-tail'
import { reportTrigger } from '../../build/output'
import getRouteFromEntrypoint from '../get-route-from-entrypoint'
import { serverComponentRegex } from '../../build/webpack/loaders/utils'
import { getPageStaticInfo } from '../../build/analysis/get-page-static-info'
import { isMiddlewareFile, isMiddlewareFilename } from '../../build/utils'
import { PageNotFoundError } from '../../shared/lib/utils'
import { DynamicParamTypesShort, FlightRouterState } from '../app-render'
import {
  CompilerNameValues,
  COMPILER_INDEXES,
  COMPILER_NAMES,
} from '../../shared/lib/constants'

const debug = origDebug('next:on-demand-entry-handler')

/**
 * Returns object keys with type inferred from the object key
 */
const keys = Object.keys as <T>(o: T) => Extract<keyof T, string>[]

const COMPILER_KEYS = keys(COMPILER_INDEXES)

function treePathToEntrypoint(
  segmentPath: string[],
  parentPath?: string
): string {
  const [parallelRouteKey, segment] = segmentPath

  // TODO-APP: modify this path to cover parallelRouteKey convention
  const path =
    (parentPath ? parentPath + '/' : '') +
    (parallelRouteKey !== 'children' ? parallelRouteKey + '/' : '') +
    (segment === '' ? 'page' : segment)

  // Last segment
  if (segmentPath.length === 2) {
    return path
  }

  const childSegmentPath = segmentPath.slice(2)
  return treePathToEntrypoint(childSegmentPath, path)
}

function convertDynamicParamTypeToSyntax(
  dynamicParamTypeShort: DynamicParamTypesShort,
  param: string
) {
  switch (dynamicParamTypeShort) {
    case 'c':
      return `[...${param}]`
    case 'oc':
      return `[[...${param}]]`
    case 'd':
      return `[${param}]`
    default:
      throw new Error('Unknown dynamic param type')
  }
}

function getEntrypointsFromTree(
  tree: FlightRouterState,
  isFirst: boolean,
  parentPath: string[] = []
) {
  const [segment, parallelRoutes] = tree

  const currentSegment = Array.isArray(segment)
    ? convertDynamicParamTypeToSyntax(segment[2], segment[0])
    : segment

  const currentPath = [...parentPath, currentSegment]

  if (!isFirst && currentSegment === '') {
    // TODO get rid of '' at the start of tree
    return [treePathToEntrypoint(currentPath.slice(1))]
  }

  return Object.keys(parallelRoutes).reduce(
    (paths: string[], key: string): string[] => {
      const childTree = parallelRoutes[key]
      const childPages = getEntrypointsFromTree(childTree, false, [
        ...currentPath,
        key,
      ])
      return [...paths, ...childPages]
    },
    []
  )
}

export const ADDED = Symbol('added')
export const BUILDING = Symbol('building')
export const BUILT = Symbol('built')

interface EntryType {
  /**
   * Tells if a page is scheduled to be disposed.
   */
  dispose?: boolean
  /**
   * Timestamp with the last time the page was active.
   */
  lastActiveTime?: number
  /**
   * Page build status.
   */
  status?: typeof ADDED | typeof BUILDING | typeof BUILT

  /**
   * Path to the page file relative to the dist folder with no extension.
   * For example: `pages/about/index`
   */
  bundlePath: string

  /**
   * Webpack request to create a dependency for.
   */
  request: string
}

// Shadowing check in ESLint does not account for enum
// eslint-disable-next-line no-shadow
export const enum EntryTypes {
  ENTRY,
  CHILD_ENTRY,
}
interface Entry extends EntryType {
  type: EntryTypes.ENTRY
  /**
   * The absolute page to the page file. Used for detecting if the file was removed. For example:
   * `/Users/Rick/project/pages/about/index.js`
   */
  absolutePagePath: string
}

interface ChildEntry extends EntryType {
  type: EntryTypes.CHILD_ENTRY
  /**
   * Which parent entries use this childEntry.
   */
  parentEntries: Set<string>
}

export const entries: {
  /**
   * The key composed of the compiler name and the page. For example:
   * `edge-server/about`
   */
  [entryName: string]: Entry | ChildEntry
} = {}

let invalidator: Invalidator
export const getInvalidator = () => invalidator

const doneCallbacks: EventEmitter | null = new EventEmitter()
const lastClientAccessPages = ['']
const lastServerAccessPagesForAppDir = ['']

type BuildingTracker = Set<CompilerNameValues>
type RebuildTracker = Set<CompilerNameValues>

// Make sure only one invalidation happens at a time
// Otherwise, webpack hash gets changed and it'll force the client to reload.
class Invalidator {
  private multiCompiler: webpack.MultiCompiler

  private building: BuildingTracker = new Set()
  private rebuildAgain: RebuildTracker = new Set()

  constructor(multiCompiler: webpack.MultiCompiler) {
    this.multiCompiler = multiCompiler
  }

  public shouldRebuildAll() {
    return this.rebuildAgain.size > 0
  }

  invalidate(compilerKeys: typeof COMPILER_KEYS = COMPILER_KEYS): void {
    for (const key of compilerKeys) {
      // If there's a current build is processing, we won't abort it by invalidating.
      // (If aborted, it'll cause a client side hard reload)
      // But let it to invalidate just after the completion.
      // So, it can re-build the queued pages at once.

      if (this.building.has(key)) {
        this.rebuildAgain.add(key)
        continue
      }

      this.multiCompiler.compilers[COMPILER_INDEXES[key]].watching?.invalidate()
      this.building.add(key)
    }
  }

  public startBuilding(compilerKey: keyof typeof COMPILER_INDEXES) {
    this.building.add(compilerKey)
  }

  public doneBuilding() {
    const rebuild: typeof COMPILER_KEYS = []
    for (const key of COMPILER_KEYS) {
      this.building.delete(key)

      if (this.rebuildAgain.has(key)) {
        rebuild.push(key)
        this.rebuildAgain.delete(key)
      }
    }
    this.invalidate(rebuild)
  }
}

function disposeInactiveEntries(maxInactiveAge: number) {
  Object.keys(entries).forEach((entryKey) => {
    const entryData = entries[entryKey]
    const { lastActiveTime, status, dispose } = entryData

    // TODO-APP: implement disposing of CHILD_ENTRY
    if (entryData.type === EntryTypes.CHILD_ENTRY) {
      return
    }

    if (dispose)
      // Skip pages already scheduled for disposing
      return

    // This means this entry is currently building or just added
    // We don't need to dispose those entries.
    if (status !== BUILT) return

    // We should not build the last accessed page even we didn't get any pings
    // Sometimes, it's possible our XHR ping to wait before completing other requests.
    // In that case, we should not dispose the current viewing page
    if (
      lastClientAccessPages.includes(entryKey) ||
      lastServerAccessPagesForAppDir.includes(entryKey)
    )
      return

    if (lastActiveTime && Date.now() - lastActiveTime > maxInactiveAge) {
      entries[entryKey].dispose = true
    }
  })
}

function tryToNormalizePagePath(page: string) {
  try {
    return normalizePagePath(page)
  } catch (err) {
    console.error(err)
    throw new PageNotFoundError(page)
  }
}

/**
 * Attempts to find a page file path from the given pages absolute directory,
 * a page and allowed extensions. If the page can't be found it will throw an
 * error. It defaults the `/_error` page to Next.js internal error page.
 *
 * @param rootDir Absolute path to the project root.
 * @param pagesDir Absolute path to the pages folder with trailing `/pages`.
 * @param normalizedPagePath The page normalized (it will be denormalized).
 * @param pageExtensions Array of page extensions.
 */
async function findPagePathData(
  rootDir: string,
  pagesDir: string,
  page: string,
  extensions: string[],
  appDir?: string
) {
  const normalizedPagePath = tryToNormalizePagePath(page)
  let pagePath: string | null = null

  if (isMiddlewareFile(normalizedPagePath)) {
    pagePath = await findPageFile(rootDir, normalizedPagePath, extensions)

    if (!pagePath) {
      throw new PageNotFoundError(normalizedPagePath)
    }

    const pageUrl = ensureLeadingSlash(
      removePagePathTail(normalizePathSep(pagePath), {
        extensions,
      })
    )

    return {
      absolutePagePath: join(rootDir, pagePath),
      bundlePath: normalizedPagePath.slice(1),
      page: posix.normalize(pageUrl),
    }
  }

  // Check appDir first falling back to pagesDir
  if (appDir) {
    pagePath = await findPageFile(appDir, normalizedPagePath, extensions)
    if (pagePath) {
      const pageUrl = ensureLeadingSlash(
        removePagePathTail(normalizePathSep(pagePath), {
          keepIndex: true,
          extensions,
        })
      )

      return {
        absolutePagePath: join(appDir, pagePath),
        bundlePath: posix.join('app', normalizePagePath(pageUrl)),
        page: posix.normalize(pageUrl),
      }
    }
  }

  if (!pagePath) {
    pagePath = await findPageFile(pagesDir, normalizedPagePath, extensions)
  }

  if (pagePath !== null) {
    const pageUrl = ensureLeadingSlash(
      removePagePathTail(normalizePathSep(pagePath), {
        extensions,
      })
    )

    return {
      absolutePagePath: join(pagesDir, pagePath),
      bundlePath: posix.join('pages', normalizePagePath(pageUrl)),
      page: posix.normalize(pageUrl),
    }
  }

  if (page === '/_error') {
    return {
      absolutePagePath: require.resolve('next/dist/pages/_error'),
      bundlePath: page,
      page: normalizePathSep(page),
    }
  } else {
    throw new PageNotFoundError(normalizedPagePath)
  }
}

export function onDemandEntryHandler({
  maxInactiveAge,
  multiCompiler,
  nextConfig,
  pagesBufferLength,
  pagesDir,
  rootDir,
  appDir,
}: {
  maxInactiveAge: number
  multiCompiler: webpack.MultiCompiler
  nextConfig: NextConfigComplete
  pagesBufferLength: number
  pagesDir: string
  rootDir: string
  appDir?: string
}) {
  invalidator = new Invalidator(multiCompiler)

  const startBuilding = (compilation: webpack.Compilation) => {
    const compilationName = compilation.name as any as CompilerNameValues
    invalidator.startBuilding(compilationName)
  }
  for (const compiler of multiCompiler.compilers) {
    compiler.hooks.make.tap('NextJsOnDemandEntries', startBuilding)
  }

  function getPagePathsFromEntrypoints(
    type: CompilerNameValues,
    entrypoints: Map<string, { name?: string }>,
    root?: boolean
  ) {
    const pagePaths: string[] = []
    for (const entrypoint of entrypoints.values()) {
      const page = getRouteFromEntrypoint(entrypoint.name!, root)
      if (page) {
        pagePaths.push(`${type}${page}`)
      } else if (
        (root && entrypoint.name === 'root') ||
        isMiddlewareFilename(entrypoint.name)
      ) {
        pagePaths.push(`${type}/${entrypoint.name}`)
      }
    }

    return pagePaths
  }

  multiCompiler.hooks.done.tap('NextJsOnDemandEntries', (multiStats) => {
    if (invalidator.shouldRebuildAll()) {
      return invalidator.doneBuilding()
    }
    const [clientStats, serverStats, edgeServerStats] = multiStats.stats
    const root = !!appDir
    const pagePaths = [
      ...getPagePathsFromEntrypoints(
        COMPILER_NAMES.client,
        clientStats.compilation.entrypoints,
        root
      ),
      ...getPagePathsFromEntrypoints(
        COMPILER_NAMES.server,
        serverStats.compilation.entrypoints,
        root
      ),
      ...(edgeServerStats
        ? getPagePathsFromEntrypoints(
            COMPILER_NAMES.edgeServer,
            edgeServerStats.compilation.entrypoints,
            root
          )
        : []),
    ]

    for (const page of pagePaths) {
      const entry = entries[page]
      if (!entry) {
        continue
      }

      if (entry.status !== BUILDING) {
        continue
      }

      entry.status = BUILT
      doneCallbacks!.emit(page)
    }

    invalidator.doneBuilding()
  })

  const pingIntervalTime = Math.max(1000, Math.min(5000, maxInactiveAge))

  setInterval(function () {
    disposeInactiveEntries(maxInactiveAge)
  }, pingIntervalTime + 1000).unref()

  function handleAppDirPing(
    tree: FlightRouterState
  ): { success: true } | { invalid: true } {
    const pages = getEntrypointsFromTree(tree, true)
    let toSend: { invalid: true } | { success: true } = { invalid: true }

    for (const page of pages) {
      for (const compilerType of [
        COMPILER_NAMES.client,
        COMPILER_NAMES.server,
        COMPILER_NAMES.edgeServer,
      ]) {
        const pageKey = `${compilerType}/${page}`
        const entryInfo = entries[pageKey]

        // If there's no entry, it may have been invalidated and needs to be re-built.
        if (!entryInfo) {
          // if (page !== lastEntry) client pings, but there's no entry for page
          continue
        }

        // We don't need to maintain active state of anything other than BUILT entries
        if (entryInfo.status !== BUILT) continue

        // If there's an entryInfo
        if (!lastServerAccessPagesForAppDir.includes(pageKey)) {
          lastServerAccessPagesForAppDir.unshift(pageKey)

          // Maintain the buffer max length
          // TODO: verify that the current pageKey is not at the end of the array as multiple entrypoints can exist
          if (lastServerAccessPagesForAppDir.length > pagesBufferLength) {
            lastServerAccessPagesForAppDir.pop()
          }
        }
        entryInfo.lastActiveTime = Date.now()
        entryInfo.dispose = false
        toSend = { success: true }
      }
    }
    return toSend
  }

  function handlePing(pg: string) {
    const page = normalizePathSep(pg)
    let toSend: { invalid: true } | { success: true } = { invalid: true }

    for (const compilerType of [
      COMPILER_NAMES.client,
      COMPILER_NAMES.server,
      COMPILER_NAMES.edgeServer,
    ]) {
      const pageKey = `${compilerType}${page}`
      const entryInfo = entries[pageKey]

      // If there's no entry, it may have been invalidated and needs to be re-built.
      if (!entryInfo) {
        // if (page !== lastEntry) client pings, but there's no entry for page
        if (compilerType === COMPILER_NAMES.client) {
          return { invalid: true }
        }
        continue
      }

      // 404 is an on demand entry but when a new page is added we have to refresh the page
      toSend = page === '/_error' ? { invalid: true } : { success: true }

      // We don't need to maintain active state of anything other than BUILT entries
      if (entryInfo.status !== BUILT) continue

      // If there's an entryInfo
      if (!lastClientAccessPages.includes(pageKey)) {
        lastClientAccessPages.unshift(pageKey)

        // Maintain the buffer max length
        if (lastClientAccessPages.length > pagesBufferLength) {
          lastClientAccessPages.pop()
        }
      }
      entryInfo.lastActiveTime = Date.now()
      entryInfo.dispose = false
    }
    return toSend
  }

  return {
    async ensurePage(page: string, clientOnly: boolean): Promise<void> {
      const stalledTime = 60
      const stalledEnsureTimeout = setTimeout(() => {
        debug(
          `Ensuring ${page} has taken longer than ${stalledTime}s, if this continues to stall this may be a bug`
        )
      }, stalledTime * 1000)

      try {
        const pagePathData = await findPagePathData(
          rootDir,
          pagesDir,
          page,
          nextConfig.pageExtensions,
          appDir
        )

        const isServerComponent = serverComponentRegex.test(
          pagePathData.absolutePagePath
        )
        const isInsideAppDir =
          appDir && pagePathData.absolutePagePath.startsWith(appDir)

        const addEntry = (
          compilerType: CompilerNameValues
        ): {
          entryKey: string
          newEntry: boolean
          shouldInvalidate: boolean
        } => {
          const entryKey = `${compilerType}${pagePathData.page}`

          if (entries[entryKey]) {
            entries[entryKey].dispose = false
            entries[entryKey].lastActiveTime = Date.now()
            if (entries[entryKey].status === BUILT) {
              return {
                entryKey,
                newEntry: false,
                shouldInvalidate: false,
              }
            }

            return {
              entryKey,
              newEntry: false,
              shouldInvalidate: true,
            }
          }

          entries[entryKey] = {
            type: EntryTypes.ENTRY,
            absolutePagePath: pagePathData.absolutePagePath,
            request: pagePathData.absolutePagePath,
            bundlePath: pagePathData.bundlePath,
            dispose: false,
            lastActiveTime: Date.now(),
            status: ADDED,
          }

          return {
            entryKey: entryKey,
            newEntry: true,
            shouldInvalidate: true,
          }
        }

        const staticInfo = await getPageStaticInfo({
          pageFilePath: pagePathData.absolutePagePath,
          nextConfig,
        })

        const added = new Map<CompilerNameValues, ReturnType<typeof addEntry>>()

        await runDependingOnPageType({
          page: pagePathData.page,
          pageRuntime: staticInfo.runtime,
          onClient: () => {
            // Skip adding the client entry for app / Server Components.
            if (isServerComponent || isInsideAppDir) {
              return
            }
            added.set(COMPILER_NAMES.client, addEntry(COMPILER_NAMES.client))
          },
          onServer: () => {
            added.set(COMPILER_NAMES.server, addEntry(COMPILER_NAMES.server))
          },
          onEdgeServer: () => {
            added.set(
              COMPILER_NAMES.edgeServer,
              addEntry(COMPILER_NAMES.edgeServer)
            )
          },
        })

        const addedValues = [...added.values()]
        const entriesThatShouldBeInvalidated = addedValues.filter(
          (entry) => entry.shouldInvalidate
        )
        const hasNewEntry = addedValues.some((entry) => entry.newEntry)

        if (hasNewEntry) {
          reportTrigger(
            !clientOnly && hasNewEntry
              ? `${pagePathData.page} (client and server)`
              : pagePathData.page
          )
        }

        if (entriesThatShouldBeInvalidated.length > 0) {
          const invalidatePromises = entriesThatShouldBeInvalidated.map(
            ({ entryKey }) => {
              return new Promise<void>((resolve, reject) => {
                doneCallbacks!.once(entryKey, (err: Error) => {
                  if (err) {
                    return reject(err)
                  }
                  resolve()
                })
              })
            }
          )
          invalidator.invalidate([...added.keys()])
          await Promise.all(invalidatePromises)
        }
      } finally {
        clearTimeout(stalledEnsureTimeout)
      }
    },

    onHMR(client: ws) {
      client.addEventListener('message', ({ data }) => {
        try {
          const parsedData = JSON.parse(
            typeof data !== 'string' ? data.toString() : data
          )

          if (parsedData.event === 'ping') {
            const result = parsedData.appDirRoute
              ? handleAppDirPing(parsedData.tree)
              : handlePing(parsedData.page)
            client.send(
              JSON.stringify({
                ...result,
                [parsedData.appDirRoute ? 'action' : 'event']: 'pong',
              })
            )
          }
        } catch (_) {}
      })
    },
  }
}