add included metadata to chunks to allow deduplicate chunk loads at r…

…untime (vercel#4488)

### Description

Chunk loading is now more than just a single chunk filepath. It might
consist of additional metadata which tells the chunk loading logic the
entry modules included in this chunk. This allows the runtime logic to
avoid loading chunks when all entry modules are already loaded.

This means loading the same chunk from multiple sources (with different
available modules) will only trigger a single load and so avoids
transferring more code than needed.

This is especially relevant in app dir where every client component is
considers as separate chunk loading.

next.js: vercel/next.js#48025

crates/turbopack-core/src/chunk/mod.rs

@@ -8,6 +8,7 @@ use std::{
     collections::HashSet,
     fmt::{Debug, Display},
     future::Future,
+    hash::Hash,
     marker::PhantomData,
 };
 
@@ -116,6 +117,21 @@ pub trait Chunk: Asset {
     }
 }
 
+/// Aggregated information about a chunk content that can be used by the runtime
+/// code to optimize chunk loading.
+#[turbo_tasks::value(shared)]
+#[derive(Default)]
+pub struct OutputChunkRuntimeInfo {
+    pub included_ids: Option<ModuleIdsVc>,
+    pub excluded_ids: Option<ModuleIdsVc>,
+    pub placeholder_for_future_extensions: (),
+}
+
+#[turbo_tasks::value_trait]
+pub trait OutputChunk: Asset {
+    fn runtime_info(&self) -> OutputChunkRuntimeInfoVc;
+}
+
 /// see [Chunk] for explanation
 #[turbo_tasks::value_trait]
 pub trait ParallelChunkReference: AssetReference + ValueToString {

crates/turbopack-css/src/asset.rs

@@ -88,6 +88,12 @@ impl CssModuleAssetVc {
         let this = self.await?;
         Ok(parse(this.source, Value::new(this.ty), this.transforms))
     }
+
+    /// Retrns the asset ident of the source without the "css" modifier
+    #[turbo_tasks::function]
+    pub async fn source_ident(self) -> Result<AssetIdentVc> {
+        Ok(self.await?.source.ident())
+    }
 }
 
 #[turbo_tasks::value_impl]

crates/turbopack-css/src/chunk/mod.rs

@@ -16,7 +16,8 @@ use turbopack_core::{
         optimize::{ChunkOptimizerVc, OptimizableChunk, OptimizableChunkVc},
         Chunk, ChunkContentResult, ChunkGroupReferenceVc, ChunkItem, ChunkItemVc, ChunkReferenceVc,
         ChunkVc, ChunkableAssetVc, ChunkingContext, ChunkingContextVc, FromChunkableAsset,
-        ModuleId, ModuleIdVc,
+        ModuleId, ModuleIdVc, ModuleIdsVc, OutputChunk, OutputChunkRuntimeInfo,
+        OutputChunkRuntimeInfoVc, OutputChunkVc,
     },
     code_builder::{CodeBuilder, CodeVc},
     ident::{AssetIdent, AssetIdentVc},
@@ -290,6 +291,24 @@ impl OptimizableChunk for CssChunk {
     }
 }
 
+#[turbo_tasks::value_impl]
+impl OutputChunk for CssChunk {
+    #[turbo_tasks::function]
+    async fn runtime_info(&self) -> Result<OutputChunkRuntimeInfoVc> {
+        let entries = self
+            .main_entries
+            .await?
+            .iter()
+            .map(|&entry| entry.as_chunk_item(self.context).id())
+            .collect();
+        Ok(OutputChunkRuntimeInfo {
+            included_ids: Some(ModuleIdsVc::cell(entries)),
+            ..Default::default()
+        }
+        .cell())
+    }
+}
+
 #[turbo_tasks::value_impl]
 impl Asset for CssChunk {
     #[turbo_tasks::function]

crates/turbopack-css/src/module_asset.rs

@@ -72,7 +72,7 @@ impl ModuleCssModuleAssetVc {
 impl Asset for ModuleCssModuleAsset {
     #[turbo_tasks::function]
     fn ident(&self) -> AssetIdentVc {
-        self.inner.ident().with_modifier(modifier())
+        self.inner.source_ident().with_modifier(modifier())
     }
 
     #[turbo_tasks::function]
@@ -418,8 +418,8 @@ struct CssProxyModuleAsset {
 #[turbo_tasks::value_impl]
 impl Asset for CssProxyModuleAsset {
     #[turbo_tasks::function]
-    fn ident(&self) -> AssetIdentVc {
-        self.module.ident()
+    async fn ident(&self) -> Result<AssetIdentVc> {
+        Ok(self.module.await?.inner.ident().with_modifier(modifier()))
     }
 
     #[turbo_tasks::function]
@@ -458,9 +458,9 @@ impl ChunkableAsset for CssProxyModuleAsset {
 #[turbo_tasks::value_impl]
 impl CssChunkPlaceable for CssProxyModuleAsset {
     #[turbo_tasks::function]
-    fn as_chunk_item(&self, context: ChunkingContextVc) -> CssChunkItemVc {
+    fn as_chunk_item(self_vc: CssProxyModuleAssetVc, context: ChunkingContextVc) -> CssChunkItemVc {
         CssProxyModuleChunkItemVc::cell(CssProxyModuleChunkItem {
-            module: self.module,
+            inner: self_vc,
             context,
         })
         .into()
@@ -482,20 +482,20 @@ impl ResolveOrigin for CssProxyModuleAsset {
 
 #[turbo_tasks::value]
 struct CssProxyModuleChunkItem {
-    module: ModuleCssModuleAssetVc,
+    inner: CssProxyModuleAssetVc,
     context: ChunkingContextVc,
 }
 
 #[turbo_tasks::value_impl]
 impl ChunkItem for CssProxyModuleChunkItem {
     #[turbo_tasks::function]
     fn asset_ident(&self) -> AssetIdentVc {
-        self.module.ident()
+        self.inner.ident()
     }
 
     #[turbo_tasks::function]
     fn references(&self) -> AssetReferencesVc {
-        self.module.references()
+        self.inner.references()
     }
 }
 
@@ -504,6 +504,8 @@ impl CssChunkItem for CssProxyModuleChunkItem {
     #[turbo_tasks::function]
     async fn content(&self) -> Result<CssChunkItemContentVc> {
         Ok(self
+            .inner
+            .await?
             .module
             .await?
             .inner

crates/turbopack-dev/js/src/runtime.dom.js

@@ -16,29 +16,38 @@ let BACKEND;
         return;
       }
 
-      const chunksToWaitFor = [];
-      for (const otherChunkPath of params.otherChunks) {
+      for (const otherChunkData of params.otherChunks) {
+        const otherChunkPath =
+          typeof otherChunkData === "string"
+            ? otherChunkData
+            : otherChunkData.path;
         if (otherChunkPath.endsWith(".css")) {
           // Mark all CSS chunks within the same chunk group as this chunk as loaded.
+          // They are just injected as <link> tag and have to way to communicate completion.
           const cssResolver = getOrCreateResolver(otherChunkPath);
           cssResolver.resolve();
         } else if (otherChunkPath.endsWith(".js")) {
-          // Only wait for JS chunks to load.
-          chunksToWaitFor.push(otherChunkPath);
+          // Chunk might have started loading, so we want to avoid triggering another load.
+          getOrCreateResolver(otherChunkPath);
         }
       }
 
-      if (params.runtimeModuleIds.length > 0) {
-        await waitForChunksToLoad(chunksToWaitFor);
+      // This waits for chunks to be loaded, but also marks included items as available.
+      await Promise.all(
+        params.otherChunks.map((otherChunkData) =>
+          loadChunk({ type: SourceTypeRuntime, chunkPath }, otherChunkData)
+        )
+      );
 
+      if (params.runtimeModuleIds.length > 0) {
         for (const moduleId of params.runtimeModuleIds) {
           getOrInstantiateRuntimeModule(moduleId, chunkPath);
         }
       }
     },
 
     loadChunk(chunkPath, source) {
-      return loadChunk(chunkPath, source);
+      return doLoadChunk(chunkPath, source);
     },
 
     unloadChunk(chunkPath) {
@@ -145,31 +154,14 @@ let BACKEND;
     chunkResolvers.delete(chunkPath);
   }
 
-  /**
-   * Waits for all provided chunks to load.
-   *
-   * @param {ChunkPath[]} chunks
-   * @returns {Promise<void>}
-   */
-  async function waitForChunksToLoad(chunks) {
-    const promises = [];
-    for (const chunkPath of chunks) {
-      const resolver = getOrCreateResolver(chunkPath);
-      if (!resolver.resolved) {
-        promises.push(resolver.promise);
-      }
-    }
-    await Promise.all(promises);
-  }
-
   /**
    * Loads the given chunk, and returns a promise that resolves once the chunk
    * has been loaded.
    *
    * @param {ChunkPath} chunkPath
    * @param {SourceInfo} source
    */
-  async function loadChunk(chunkPath, source) {
+  async function doLoadChunk(chunkPath, source) {
     const resolver = getOrCreateResolver(chunkPath);
     if (resolver.resolved) {
       return resolver.promise;

crates/turbopack-dev/js/src/runtime.js

@@ -9,6 +9,7 @@
 /** @typedef {import('../types').ChunkList} ChunkList */
 
 /** @typedef {import('../types').Module} Module */
+/** @typedef {import('../types').ChunkData} ChunkData */
 /** @typedef {import('../types').SourceInfo} SourceInfo */
 /** @typedef {import('../types').SourceType} SourceType */
 /** @typedef {import('../types').SourceType.Runtime} SourceTypeRuntime */
@@ -209,12 +210,45 @@ externalRequire.resolve = (name, opt) => {
   return require.resolve(name, opt);
 };
 
+/** @type {Map<ModuleId, Promise<any> | true>} */
+const availableModules = new Map();
+
+/**
+ * @param {SourceInfo} source
+ * @param {ChunkData} chunkData
+ * @returns {Promise<any>}
+ */
+async function loadChunk(source, chunkData) {
+  if (typeof chunkData === "string") {
+    return loadChunkPath(source, chunkData);
+  } else {
+    const includedList = chunkData.included || [];
+    const promises = includedList.map((included) => {
+      if (moduleFactories[included]) return true;
+      return availableModules.get(included);
+    });
+    if (promises.length > 0 && promises.every((p) => p)) {
+      // When all included items are already loaded or loading, we can skip loading ourselves
+      return Promise.all(promises);
+    }
+    const promise = loadChunkPath(source, chunkData.path);
+    for (const included of includedList) {
+      if (!availableModules.has(included)) {
+        // It might be better to race old and new promises, but it's rare that the new promise will be faster than a request started earlier.
+        // In production it's even more rare, because the chunk optimization tries to deduplicate modules anyway.
+        availableModules.set(included, promise);
+      }
+    }
+    return promise;
+  }
+}
+
 /**
  * @param {SourceInfo} source
- * @param {string} chunkPath
- * @returns {Promise<any> | undefined}
+ * @param {ChunkPath} chunkPath
+ * @returns {Promise<any>}
  */
-async function loadChunk(source, chunkPath) {
+async function loadChunkPath(source, chunkPath) {
   try {
     await BACKEND.loadChunk(chunkPath, source);
   } catch (error) {
@@ -1208,6 +1242,7 @@ function disposeChunk(chunkPath) {
     if (noRemainingChunks) {
       moduleChunksMap.delete(moduleId);
       disposeModule(moduleId, "clear");
+      availableModules.delete(moduleId);
     }
   }
 
@@ -1253,7 +1288,11 @@ function registerChunkList(chunkList) {
   ]);
 
   // Adding chunks to chunk lists and vice versa.
-  const chunks = new Set(chunkList.chunks);
+  const chunks = new Set(
+    chunkList.chunks.map((chunkData) =>
+      typeof chunkData === "string" ? chunkData : chunkData.path
+    )
+  );
   chunkListChunksMap.set(chunkList.path, chunks);
   for (const chunkPath of chunks) {
     let chunkChunkLists = chunkChunkListsMap.get(chunkPath);

crates/turbopack-dev/js/src/runtime.nodejs.js

@@ -13,11 +13,16 @@ let BACKEND;
       }
 
       if (params.runtimeModuleIds.length > 0) {
-        for (const otherChunkPath of params.otherChunks) {
-          loadChunk(otherChunkPath, {
-            type: SourceTypeRuntime,
-            chunkPath,
-          });
+        for (const otherChunkData of params.otherChunks) {
+          loadChunk(
+            typeof otherChunkData === "string"
+              ? otherChunkData
+              : otherChunkData.path,
+            {
+              type: SourceTypeRuntime,
+              chunkPath,
+            }
+          );
         }
 
         for (const moduleId of params.runtimeModuleIds) {

crates/turbopack-dev/js/src/runtime.none.js

@@ -2,6 +2,7 @@
 /** @typedef {import('../types/runtime.none').ChunkRunner} ChunkRunner */
 /** @typedef {import('../types').ModuleId} ModuleId */
 /** @typedef {import('../types').ChunkPath} ChunkPath */
+/** @typedef {import('../types').ChunkData} ChunkData */
 
 /** @type {RuntimeBackend} */
 let BACKEND;
@@ -55,7 +56,7 @@ let BACKEND;
    * dependencies of the chunk have been registered.
    *
    * @param {ChunkPath} chunkPath
-   * @param {ChunkPath[]} otherChunks
+   * @param {ChunkData[]} otherChunks
    * @param {ModuleId[]} runtimeModuleIds
    */
   function registerChunkRunner(chunkPath, otherChunks, runtimeModuleIds) {
@@ -66,7 +67,11 @@ let BACKEND;
       requiredChunks,
     };
 
-    for (const otherChunkPath of otherChunks) {
+    for (const otherChunkData of otherChunks) {
+      const otherChunkPath =
+        typeof otherChunkData === "string"
+          ? otherChunkData
+          : otherChunkData.path;
       if (registeredChunks.has(otherChunkPath)) {
         continue;
       }

crates/turbopack-dev/js/types/backend.d.ts

@@ -1,6 +1,7 @@
 import {
   GetFirstModuleChunk,
   GetOrInstantiateRuntimeModule,
+  LoadChunk,
   SourceType,
 } from ".";
 import { DevRuntimeParams } from "./runtime";
@@ -11,6 +12,7 @@ declare global {
   declare const RUNTIME_PARAMS: DevRuntimeParams;
   declare const getFirstModuleChunk: GetFirstModuleChunk;
   declare const getOrInstantiateRuntimeModule: GetOrInstantiateRuntimeModule;
+  declare const loadChunk: InternalLoadChunk;
   declare const SourceTypeRuntime: SourceType.Runtime;
   declare const SourceTypeParent: SourceType.Parent;
   declare const SourceTypeUpdate: SourceType.Update;

crates/turbopack-dev/js/types/index.d.ts

@@ -22,9 +22,16 @@ export type ChunkRegistration = [
   chunkModules: ChunkModule[],
   DevRuntimeParams | undefined
 ];
+export type ChunkData =
+  | ChunkPath
+  | {
+      path: ChunkPath;
+      included: ModuleId[];
+      excluded: ModuleId[];
+    };
 export type ChunkList = {
   path: ChunkPath;
-  chunks: ChunkPath[];
+  chunks: ChunkData[];
   source: "entry" | "dynamic";
 };
 
@@ -134,6 +141,10 @@ export type GetOrInstantiateRuntimeModule = (
   moduleId: ModuleId,
   chunkPath: ChunkPath
 ) => Module;
+export type InternalLoadChunk = (
+  source: SourceInfo,
+  chunkData: ChunkData
+) => Promise<any>;
 
 export interface Loader {
   promise: Promise<undefined>;