allow to create internal modules via AssetContext (#5095)

### Description

This allows to create an Module with inner asset via AssetContext.

This will also respect transitions.

This change allows use to use `context.process(source,
ReferenceType::Internal(inner_assets)` instead of
`EcmascriptModuleAssetVc::new_with_inner_assets(..., ..., ..., ..., ...,
inner_assets)`

next.js PR: vercel/next.js#50338

crates/turbopack-core/src/reference_type.rs

@@ -1,6 +1,19 @@
 use std::fmt::Display;
 
-use crate::resolve::ModulePartVc;
+use indexmap::IndexMap;
+
+use crate::{asset::AssetVc, resolve::ModulePartVc};
+
+#[turbo_tasks::value(transparent)]
+pub struct InnerAssets(IndexMap<String, AssetVc>);
+
+#[turbo_tasks::value_impl]
+impl InnerAssetsVc {
+    #[turbo_tasks::function]
+    pub fn empty() -> Self {
+        InnerAssetsVc::cell(IndexMap::new())
+    }
+}
 
 // These enums list well-known types, which we use internally. Plugins might add
 // custom types too.
@@ -72,6 +85,7 @@ pub enum ReferenceType {
     Url(UrlReferenceSubType),
     TypeScript(TypeScriptReferenceSubType),
     Entry(EntryReferenceSubType),
+    Internal(InnerAssetsVc),
     Custom(u8),
     Undefined,
 }
@@ -89,6 +103,7 @@ impl Display for ReferenceType {
             ReferenceType::Url(_) => "url",
             ReferenceType::TypeScript(_) => "typescript",
             ReferenceType::Entry(_) => "entry",
+            ReferenceType::Internal(_) => "internal",
             ReferenceType::Custom(_) => todo!(),
             ReferenceType::Undefined => "undefined",
         };
@@ -126,6 +141,7 @@ impl ReferenceType {
                 matches!(other, ReferenceType::Entry(_))
                     && matches!(sub_type, EntryReferenceSubType::Undefined)
             }
+            ReferenceType::Internal(_) => matches!(other, ReferenceType::Internal(_)),
             ReferenceType::Custom(_) => {
                 todo!()
             }

crates/turbopack-ecmascript/src/lib.rs

@@ -31,7 +31,6 @@ use chunk::{
     EcmascriptChunkingContextVc,
 };
 use code_gen::CodeGenerateableVc;
-use indexmap::IndexMap;
 use parse::{parse, ParseResult};
 pub use parse::{ParseResultSourceMap, ParseResultSourceMapVc};
 use path_visitor::ApplyVisitors;
@@ -64,9 +63,11 @@ use turbopack_core::{
     context::AssetContextVc,
     ident::AssetIdentVc,
     reference::{AssetReferencesReadRef, AssetReferencesVc},
+    reference_type::InnerAssetsVc,
     resolve::{
         origin::{ResolveOrigin, ResolveOriginVc},
         parse::RequestVc,
+        ModulePartVc,
     },
 };
 
@@ -80,6 +81,7 @@ use self::{
         VisitorFactory,
     },
     parse::ParseResultVc,
+    tree_shake::asset::EcmascriptModulePartAssetVc,
 };
 use crate::{
     chunk::{EcmascriptChunkPlaceable, EcmascriptChunkPlaceableVc},
@@ -122,9 +124,6 @@ pub enum EcmascriptModuleAssetType {
     TypescriptDeclaration,
 }
 
-#[turbo_tasks::value(transparent)]
-pub struct InnerAssets(IndexMap<String, AssetVc>);
-
 #[turbo_tasks::function]
 fn modifier() -> StringVc {
     StringVc::cell("ecmascript".to_string())
@@ -137,6 +136,62 @@ struct MemoizedSuccessfulAnalysis {
     exports: EcmascriptExportsReadRef,
 }
 
+pub struct EcmascriptModuleAssetBuilder {
+    source: AssetVc,
+    context: AssetContextVc,
+    ty: EcmascriptModuleAssetType,
+    transforms: EcmascriptInputTransformsVc,
+    options: EcmascriptOptions,
+    compile_time_info: CompileTimeInfoVc,
+    inner_assets: Option<InnerAssetsVc>,
+    part: Option<ModulePartVc>,
+}
+
+impl EcmascriptModuleAssetBuilder {
+    pub fn with_inner_assets(mut self, inner_assets: InnerAssetsVc) -> Self {
+        self.inner_assets = Some(inner_assets);
+        self
+    }
+
+    pub fn with_type(mut self, ty: EcmascriptModuleAssetType) -> Self {
+        self.ty = ty;
+        self
+    }
+
+    pub fn with_part(mut self, part: ModulePartVc) -> Self {
+        self.part = Some(part);
+        self
+    }
+
+    pub fn build(self) -> AssetVc {
+        let base = if let Some(inner_assets) = self.inner_assets {
+            EcmascriptModuleAssetVc::new_with_inner_assets(
+                self.source,
+                self.context,
+                Value::new(self.ty),
+                self.transforms,
+                Value::new(self.options),
+                self.compile_time_info,
+                inner_assets,
+            )
+        } else {
+            EcmascriptModuleAssetVc::new(
+                self.source,
+                self.context,
+                Value::new(self.ty),
+                self.transforms,
+                Value::new(self.options),
+                self.compile_time_info,
+            )
+        };
+        if let Some(part) = self.part {
+            EcmascriptModulePartAssetVc::new(base, part).into()
+        } else {
+            base.into()
+        }
+    }
+}
+
 #[turbo_tasks::value]
 pub struct EcmascriptModuleAsset {
     pub source: AssetVc,
@@ -155,6 +210,27 @@ pub struct EcmascriptModuleAsset {
 #[turbo_tasks::value(transparent)]
 pub struct OptionEcmascriptModuleAsset(Option<EcmascriptModuleAssetVc>);
 
+impl EcmascriptModuleAssetVc {
+    pub fn builder(
+        source: AssetVc,
+        context: AssetContextVc,
+        transforms: EcmascriptInputTransformsVc,
+        options: EcmascriptOptions,
+        compile_time_info: CompileTimeInfoVc,
+    ) -> EcmascriptModuleAssetBuilder {
+        EcmascriptModuleAssetBuilder {
+            source,
+            context,
+            ty: EcmascriptModuleAssetType::Ecmascript,
+            transforms,
+            options,
+            compile_time_info,
+            inner_assets: None,
+            part: None,
+        }
+    }
+}
+
 #[turbo_tasks::value_impl]
 impl EcmascriptModuleAssetVc {
     #[turbo_tasks::function]

crates/turbopack-ecmascript/src/tree_shake/asset.rs

@@ -30,16 +30,18 @@ pub struct EcmascriptModulePartAsset {
     pub(crate) part: ModulePartVc,
 }
 
+#[turbo_tasks::value_impl]
 impl EcmascriptModulePartAssetVc {
     /// Create a new instance of [EcmascriptModulePartAssetVc], whcih consists
     /// of a pointer to the full module and the [ModulePart] pointing the part
     /// of the module.
-    pub fn new(module: EcmascriptModuleAssetVc, part: ModulePartVc) -> Result<Self> {
-        Ok(EcmascriptModulePartAsset {
+    #[turbo_tasks::function]
+    pub fn new(module: EcmascriptModuleAssetVc, part: ModulePartVc) -> Self {
+        EcmascriptModulePartAsset {
             full_module: module,
             part,
         }
-        .cell())
+        .cell()
     }
 }
 
@@ -79,7 +81,7 @@ impl Asset for EcmascriptModulePartAsset {
                     EcmascriptModulePartAssetVc::new(
                         self.full_module,
                         ModulePartVc::internal(part_id),
-                    )?
+                    )
                     .as_asset(),
                     StringVc::cell("ecmascript module part".to_string()),
                 )

crates/turbopack-node/src/evaluate.rs

@@ -1,6 +1,6 @@
 use std::{borrow::Cow, ops::ControlFlow, thread::available_parallelism, time::Duration};
 
-use anyhow::{anyhow, Result};
+use anyhow::{anyhow, bail, Result};
 use async_stream::try_stream as generator;
 use futures::{
     channel::mpsc::{unbounded, UnboundedSender},
@@ -24,17 +24,16 @@ use turbo_tasks_fs::{
 };
 use turbopack_core::{
     asset::{Asset, AssetVc},
-    chunk::{ChunkableAsset, ChunkingContext, ChunkingContextVc, EvaluatableAssetsVc},
+    chunk::{
+        ChunkableAsset, ChunkingContext, ChunkingContextVc, EvaluatableAssetVc, EvaluatableAssetsVc,
+    },
     context::{AssetContext, AssetContextVc},
     ident::AssetIdentVc,
     issue::{Issue, IssueSeverity, IssueSeverityVc, IssueVc},
+    reference_type::{InnerAssetsVc, ReferenceType},
     source_asset::SourceAssetVc,
     virtual_asset::VirtualAssetVc,
 };
-use turbopack_ecmascript::{
-    EcmascriptInputTransform, EcmascriptInputTransformsVc, EcmascriptModuleAssetType,
-    EcmascriptModuleAssetVc, InnerAssetsVc,
-};
 
 use crate::{
     bootstrap::NodeJsBootstrapAsset,
@@ -108,17 +107,10 @@ pub async fn get_evaluate_pool(
     additional_invalidation: CompletionVc,
     debug: bool,
 ) -> Result<NodeJsPoolVc> {
-    let runtime_asset = EcmascriptModuleAssetVc::new(
+    let runtime_asset = context.process(
         SourceAssetVc::new(embed_file_path("ipc/evaluate.ts")).into(),
-        context,
-        Value::new(EcmascriptModuleAssetType::Typescript),
-        EcmascriptInputTransformsVc::cell(vec![EcmascriptInputTransform::TypeScript {
-            use_define_for_class_fields: false,
-        }]),
-        Value::new(Default::default()),
-        context.compile_time_info(),
-    )
-    .as_asset();
+        Value::new(ReferenceType::Internal(InnerAssetsVc::empty())),
+    );
 
     let module_path = module_asset.ident().path().await?;
     let file_name = module_path.file_name();
@@ -130,7 +122,7 @@ pub async fn get_evaluate_pool(
         Cow::Owned(format!("{file_name}.js"))
     };
     let path = chunking_context.output_root().join(file_name.as_ref());
-    let entry_module = EcmascriptModuleAssetVc::new_with_inner_assets(
+    let entry_module = context.process(
         VirtualAssetVc::new(
             runtime_asset.ident().path().join("evaluate.js"),
             File::from(
@@ -140,40 +132,34 @@ pub async fn get_evaluate_pool(
             .into(),
         )
         .into(),
-        context,
-        Value::new(EcmascriptModuleAssetType::Typescript),
-        EcmascriptInputTransformsVc::cell(vec![EcmascriptInputTransform::TypeScript {
-            use_define_for_class_fields: false,
-        }]),
-        Value::new(Default::default()),
-        context.compile_time_info(),
-        InnerAssetsVc::cell(indexmap! {
+        Value::new(ReferenceType::Internal(InnerAssetsVc::cell(indexmap! {
             "INNER".to_string() => module_asset,
             "RUNTIME".to_string() => runtime_asset
-        }),
+        }))),
     );
 
+    let Some(entry_module) = EvaluatableAssetVc::resolve_from(entry_module).await? else {
+        bail!("Internal module is not evaluatable");
+    };
+
     let (Some(cwd), Some(entrypoint)) = (to_sys_path(cwd).await?, to_sys_path(path).await?) else {
         panic!("can only evaluate from a disk filesystem");
     };
 
     let runtime_entries = {
-        let globals_module = EcmascriptModuleAssetVc::new(
+        let globals_module = context.process(
             SourceAssetVc::new(embed_file_path("globals.ts")).into(),
-            context,
-            Value::new(EcmascriptModuleAssetType::Typescript),
-            EcmascriptInputTransformsVc::cell(vec![EcmascriptInputTransform::TypeScript {
-                use_define_for_class_fields: false,
-            }]),
-            Value::new(Default::default()),
-            context.compile_time_info(),
-        )
-        .as_evaluatable_asset();
+            Value::new(ReferenceType::Internal(InnerAssetsVc::empty())),
+        );
+
+        let Some(globals_module) = EvaluatableAssetVc::resolve_from(globals_module).await? else {
+            bail!("Internal module is not evaluatable");
+        };
 
         let mut entries = vec![globals_module];
         if let Some(runtime_entries) = runtime_entries {
-            for entry in &*runtime_entries.await? {
-                entries.push(*entry)
+            for &entry in &*runtime_entries.await? {
+                entries.push(entry)
             }
         }
 
@@ -184,7 +170,7 @@ pub async fn get_evaluate_pool(
         path,
         chunking_context,
         entry: entry_module.as_root_chunk(chunking_context),
-        evaluatable_assets: runtime_entries.with_entry(entry_module.into()),
+        evaluatable_assets: runtime_entries.with_entry(entry_module),
     }
     .cell()
     .into();

crates/turbopack-node/src/lib.rs

@@ -24,7 +24,6 @@ use turbopack_core::{
     source_map::GenerateSourceMapVc,
     virtual_asset::VirtualAssetVc,
 };
-use turbopack_ecmascript::EcmascriptModuleAssetVc;
 
 use self::{
     bootstrap::NodeJsBootstrapAsset,
@@ -114,13 +113,13 @@ async fn internal_assets_for_source_mapping(
 /// subgraph
 #[turbo_tasks::function]
 pub async fn external_asset_entrypoints(
-    module: EcmascriptModuleAssetVc,
+    module: EvaluatableAssetVc,
     runtime_entries: EvaluatableAssetsVc,
     chunking_context: ChunkingContextVc,
     intermediate_output_path: FileSystemPathVc,
 ) -> Result<AssetsSetVc> {
     Ok(separate_assets(
-        get_intermediate_asset(chunking_context, module.into(), runtime_entries)
+        get_intermediate_asset(chunking_context, module, runtime_entries)
             .resolve()
             .await?,
         intermediate_output_path,

crates/turbopack-node/src/node_entry.rs

@@ -1,14 +1,13 @@
 use anyhow::Result;
 use turbo_tasks::Value;
 use turbo_tasks_fs::FileSystemPathVc;
-use turbopack_core::chunk::{ChunkingContextVc, EvaluatableAssetsVc};
+use turbopack_core::chunk::{ChunkingContextVc, EvaluatableAssetVc, EvaluatableAssetsVc};
 use turbopack_dev_server::source::ContentSourceData;
-use turbopack_ecmascript::EcmascriptModuleAssetVc;
 
 #[turbo_tasks::value(shared)]
 pub struct NodeRenderingEntry {
     pub runtime_entries: EvaluatableAssetsVc,
-    pub module: EcmascriptModuleAssetVc,
+    pub module: EvaluatableAssetVc,
     pub chunking_context: ChunkingContextVc,
     pub intermediate_output_path: FileSystemPathVc,
     pub output_root: FileSystemPathVc,