From 3860eca0808ef0429a995b36d0fee2fdd57fc1da Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Tue, 8 Feb 2022 18:37:26 -0800 Subject: [PATCH 01/87] internalization bug --- .../experimental/src/ExperimentalBundler.js | 28 +++++++++++++++++-- packages/configs/default/index.json | 2 +- .../index.html | 6 ++++ .../integration-tests/test/scope-hoisting.js | 4 +-- 4 files changed, 34 insertions(+), 6 deletions(-) create mode 100644 packages/core/integration-tests/test/integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.html diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 99719dba6c6..60f90c34675 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -186,6 +186,7 @@ function decorateLegacyGraph( for (let [, idealBundle] of idealBundleGraph.nodes) { if (idealBundle === 'root') continue; let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); + console.log(idealBundle.internalizedAssetIds); for (let internalized of idealBundle.internalizedAssetIds) { let incomingDeps = bundleGraph.getIncomingDependencies( bundleGraph.getAssetById(internalized), @@ -522,7 +523,7 @@ function createIdealGraph( if (node.value === root) { return; } - + debugger; if (node.type === 'dependency') { let dependency = node.value; @@ -560,6 +561,19 @@ function createIdealGraph( if (bundleRoots.has(node.value)) { actions.skipChildren(); + for (let bundleGroupId of bundleGraph.getNodeIdsConnectedTo( + nullthrows(bundles.get(root.id)), + )) { + console.log( + 'connecting bundlegroup', + bundleGroupId, + bundles.get(node.value.id), + ); + bundleGraph.addEdge( + bundleGroupId, + nullthrows(bundles.get(node.value.id)), + ); + } return; } @@ -570,7 +584,8 @@ function createIdealGraph( reachableRoots.addEdge(rootNodeId, nodeId); }, root); } - + reachableRoots; + debugger; // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets let ancestorAssets: Map< @@ -589,6 +604,7 @@ function createIdealGraph( const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); + debugger; let ancestors = ancestorAssets.get(bundleRoot); // First consider bundle group asset availability, processing only @@ -617,7 +633,7 @@ function createIdealGraph( ) .map(id => nullthrows(reachableRoots.getNode(id))); - for (let asset of assetsFromBundleRoot) { + for (let asset of [bundleRoot, ...assetsFromBundleRoot]) { assetRefs.set(asset, assetRefs.get(asset) + 1); } } @@ -743,6 +759,10 @@ function createIdealGraph( return false; } + console.log('checking br', bundleRoot.filePath); + console.log('checking asset', asset.filePath); + debugger; + return ( reachableRoots.hasEdge( reachableRoots.getNodeIdByContentKey(bundleRoot.id), @@ -756,6 +776,8 @@ function createIdealGraph( return bundleRoot; }); + console.log(willInternalizeRoots); + for (let bundleRoot of willInternalizeRoots) { if (bundleRoot !== asset) { let bundle = nullthrows( diff --git a/packages/configs/default/index.json b/packages/configs/default/index.json index bda82bd3089..ac78954b0c1 100644 --- a/packages/configs/default/index.json +++ b/packages/configs/default/index.json @@ -1,5 +1,5 @@ { - "bundler": "@parcel/bundler-default", + "bundler": "@parcel/bundler-experimental", "transformers": { "types:*.{ts,tsx}": ["@parcel/transformer-typescript-types"], "bundle-text:*": ["...", "@parcel/transformer-inline-string"], diff --git a/packages/core/integration-tests/test/integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.html b/packages/core/integration-tests/test/integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.html new file mode 100644 index 00000000000..233f9c37633 --- /dev/null +++ b/packages/core/integration-tests/test/integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.html @@ -0,0 +1,6 @@ + + + + + + diff --git a/packages/core/integration-tests/test/scope-hoisting.js b/packages/core/integration-tests/test/scope-hoisting.js index b1632bfc83e..1cc4537a86f 100644 --- a/packages/core/integration-tests/test/scope-hoisting.js +++ b/packages/core/integration-tests/test/scope-hoisting.js @@ -3019,11 +3019,11 @@ describe('scope hoisting', function () { assert.deepEqual(await run(b), 42); }); - it('individually exports symbols from intermediately wrapped reexports', async () => { + it.only('individually exports symbols from intermediately wrapped reexports', async () => { let b = await bundle( path.join( __dirname, - 'integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.mjs', + 'integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.html', ), ); From 406cacb2af6ad32ff2644b395de61014780152ea Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 9 Feb 2022 12:07:37 -0800 Subject: [PATCH 02/87] Cleanup --- .../experimental/src/ExperimentalBundler.js | 39 +++++++------------ .../integration-tests/test/scope-hoisting.js | 3 +- 2 files changed, 15 insertions(+), 27 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 60f90c34675..b94a2eb2800 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -186,7 +186,6 @@ function decorateLegacyGraph( for (let [, idealBundle] of idealBundleGraph.nodes) { if (idealBundle === 'root') continue; let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); - console.log(idealBundle.internalizedAssetIds); for (let internalized of idealBundle.internalizedAssetIds) { let incomingDeps = bundleGraph.getIncomingDependencies( bundleGraph.getAssetById(internalized), @@ -252,11 +251,10 @@ function createIdealGraph( Array<[Dependency, Bundle]>, > = new DefaultMap(() => []); - // bundleRoot to all bundleRoot descendants - let reachableBundles: DefaultMap< - BundleRoot, - Set, - > = new DefaultMap(() => new Set()); + // Connects bundleRoot assets to the assets that must be in the bundle + let reachableBundles: DefaultMap> = new DefaultMap( + () => new Set(), + ); let bundleGraph: Graph = new Graph(); let stack: Array<[BundleRoot, NodeId]> = []; @@ -523,7 +521,7 @@ function createIdealGraph( if (node.value === root) { return; } - debugger; + if (node.type === 'dependency') { let dependency = node.value; @@ -561,19 +559,14 @@ function createIdealGraph( if (bundleRoots.has(node.value)) { actions.skipChildren(); - for (let bundleGroupId of bundleGraph.getNodeIdsConnectedTo( - nullthrows(bundles.get(root.id)), - )) { - console.log( - 'connecting bundlegroup', - bundleGroupId, - bundles.get(node.value.id), - ); - bundleGraph.addEdge( - bundleGroupId, - nullthrows(bundles.get(node.value.id)), - ); - } + // for (let bundleGroupId of bundleGraph.getNodeIdsConnectedTo( + // nullthrows(bundles.get(root.id)), + // )) { + // bundleGraph.addEdge( + // bundleGroupId, + // nullthrows(bundles.get(node.value.id)), + // ); + // } return; } @@ -759,10 +752,6 @@ function createIdealGraph( return false; } - console.log('checking br', bundleRoot.filePath); - console.log('checking asset', asset.filePath); - debugger; - return ( reachableRoots.hasEdge( reachableRoots.getNodeIdByContentKey(bundleRoot.id), @@ -776,8 +765,6 @@ function createIdealGraph( return bundleRoot; }); - console.log(willInternalizeRoots); - for (let bundleRoot of willInternalizeRoots) { if (bundleRoot !== asset) { let bundle = nullthrows( diff --git a/packages/core/integration-tests/test/scope-hoisting.js b/packages/core/integration-tests/test/scope-hoisting.js index 1cc4537a86f..20cf1b8a34c 100644 --- a/packages/core/integration-tests/test/scope-hoisting.js +++ b/packages/core/integration-tests/test/scope-hoisting.js @@ -14,6 +14,7 @@ import { getNextBuild, mergeParcelOptions, outputFS, + inputFS, overlayFS, run, runBundle, @@ -3023,7 +3024,7 @@ describe('scope hoisting', function () { let b = await bundle( path.join( __dirname, - 'integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.html', + 'integration/scope-hoisting/es6/export-intermediate-wrapped-reexports/index.mjs', ), ); From c69e6d22b91416f699835bb280acf9cf8910a348 Mon Sep 17 00:00:00 2001 From: Eric Eldredge Date: Wed, 9 Feb 2022 17:21:04 -0500 Subject: [PATCH 03/87] Always add assets to reachable bundles --- .../experimental/src/ExperimentalBundler.js | 24 +++++++------------ .../integration-tests/test/scope-hoisting.js | 1 - 2 files changed, 8 insertions(+), 17 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index b94a2eb2800..e1cf15dc32b 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -532,7 +532,7 @@ function createIdealGraph( ) { let assets = assetGraph.getDependencyAssets(dependency); if (assets.length === 0) { - return node; + return dependency.priority === 'lazy'; } invariant(assets.length === 1); @@ -552,21 +552,15 @@ function createIdealGraph( ); } } - return; } - return; + return dependency.priority === 'lazy'; } - if (bundleRoots.has(node.value)) { + if ( + bundleRoots.has(node.value) && + (!node.value.isBundleSplittable || isAsync) + ) { actions.skipChildren(); - // for (let bundleGroupId of bundleGraph.getNodeIdsConnectedTo( - // nullthrows(bundles.get(root.id)), - // )) { - // bundleGraph.addEdge( - // bundleGroupId, - // nullthrows(bundles.get(node.value.id)), - // ); - // } return; } @@ -577,8 +571,6 @@ function createIdealGraph( reachableRoots.addEdge(rootNodeId, nodeId); }, root); } - reachableRoots; - debugger; // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets let ancestorAssets: Map< @@ -597,7 +589,6 @@ function createIdealGraph( const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); - debugger; let ancestors = ancestorAssets.get(bundleRoot); // First consider bundle group asset availability, processing only @@ -775,7 +766,8 @@ function createIdealGraph( } } } - } else if (reachable.length > 0) { + } + if (reachable.length > 0) { let reachableEntries = reachable.filter( a => entries.has(a) || !a.isBundleSplittable, ); diff --git a/packages/core/integration-tests/test/scope-hoisting.js b/packages/core/integration-tests/test/scope-hoisting.js index 20cf1b8a34c..044d41f98f1 100644 --- a/packages/core/integration-tests/test/scope-hoisting.js +++ b/packages/core/integration-tests/test/scope-hoisting.js @@ -14,7 +14,6 @@ import { getNextBuild, mergeParcelOptions, outputFS, - inputFS, overlayFS, run, runBundle, From 51ab20fee99a367180b551dffe94e84e5b11f908 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 9 Feb 2022 15:19:29 -0800 Subject: [PATCH 04/87] cleanup --- packages/bundlers/experimental/src/ExperimentalBundler.js | 3 +++ packages/core/integration-tests/test/scope-hoisting.js | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e1cf15dc32b..e5b96ac1cd7 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -182,6 +182,7 @@ function decorateLegacyGraph( bundleGraph.addAssetToBundle(asset, bundle); } } + // Step 2: Internalize dependencies for bundles for (let [, idealBundle] of idealBundleGraph.nodes) { if (idealBundle === 'root') continue; @@ -200,6 +201,7 @@ function decorateLegacyGraph( } } } + // Step 3: Add bundles to their bundle groups for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); @@ -571,6 +573,7 @@ function createIdealGraph( reachableRoots.addEdge(rootNodeId, nodeId); }, root); } + // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets let ancestorAssets: Map< diff --git a/packages/core/integration-tests/test/scope-hoisting.js b/packages/core/integration-tests/test/scope-hoisting.js index 044d41f98f1..b1632bfc83e 100644 --- a/packages/core/integration-tests/test/scope-hoisting.js +++ b/packages/core/integration-tests/test/scope-hoisting.js @@ -3019,7 +3019,7 @@ describe('scope hoisting', function () { assert.deepEqual(await run(b), 42); }); - it.only('individually exports symbols from intermediately wrapped reexports', async () => { + it('individually exports symbols from intermediately wrapped reexports', async () => { let b = await bundle( path.join( __dirname, From febecdebdad450e5736465e4f45d6816622106f7 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 9 Feb 2022 16:59:29 -0800 Subject: [PATCH 05/87] fixed test regressions --- .../experimental/src/ExperimentalBundler.js | 13 ++++++------- packages/configs/default/index.json | 2 +- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e5b96ac1cd7..87839b157fe 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -528,13 +528,10 @@ function createIdealGraph( let dependency = node.value; if (dependencyBundleGraph.hasContentKey(dependency.id)) { - if ( - dependency.priority === 'lazy' || - dependency.priority === 'parallel' - ) { + if (dependency.priority !== 'sync') { let assets = assetGraph.getDependencyAssets(dependency); if (assets.length === 0) { - return dependency.priority === 'lazy'; + return true; } invariant(assets.length === 1); @@ -555,12 +552,14 @@ function createIdealGraph( } } } - return dependency.priority === 'lazy'; + return dependency.priority !== 'sync'; } if ( bundleRoots.has(node.value) && - (!node.value.isBundleSplittable || isAsync) + ((root.isBundleSplittable && !entries.has(root)) || + isAsync || + root.type !== node.value.type) ) { actions.skipChildren(); return; diff --git a/packages/configs/default/index.json b/packages/configs/default/index.json index ac78954b0c1..bda82bd3089 100644 --- a/packages/configs/default/index.json +++ b/packages/configs/default/index.json @@ -1,5 +1,5 @@ { - "bundler": "@parcel/bundler-experimental", + "bundler": "@parcel/bundler-default", "transformers": { "types:*.{ts,tsx}": ["@parcel/transformer-typescript-types"], "bundle-text:*": ["...", "@parcel/transformer-inline-string"], From c403cdaf9361df5addf2623dcf0451b3e6b0b4b7 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Thu, 10 Feb 2022 11:47:21 -0800 Subject: [PATCH 06/87] add ref edges whenever there are bundle edges --- .../bundlers/experimental/src/ExperimentalBundler.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 87839b157fe..c0c884c7b2c 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -205,6 +205,11 @@ function decorateLegacyGraph( // Step 3: Add bundles to their bundle groups for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); + let entryBundle = nullthrows(idealBundleGraph.getNode(bundleId)); + invariant(entryBundle !== 'root'); + let legacyEntryBundle = nullthrows( + idealBundleToLegacyBundle.get(entryBundle), + ); for (let id of outboundNodeIds) { let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); invariant(siblingBundle !== 'root'); @@ -212,6 +217,7 @@ function decorateLegacyGraph( idealBundleToLegacyBundle.get(siblingBundle), ); bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); + bundleGraph.createBundleReference(legacyEntryBundle, legacySiblingBundle); } } @@ -561,6 +567,12 @@ function createIdealGraph( isAsync || root.type !== node.value.type) ) { + if (!isAsync) { + bundleGraph.addEdge( + nullthrows(bundles.get(root.id)), + nullthrows(bundles.get(node.value.id)), + ); + } actions.skipChildren(); return; } From 8ac07019931012c33088c9b194be6231f80b4b37 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Mon, 14 Feb 2022 17:59:10 -0800 Subject: [PATCH 07/87] add edge from bundlegroups to bundles wip --- .../experimental/src/ExperimentalBundler.js | 43 ++++++++++++++++--- 1 file changed, 38 insertions(+), 5 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index c0c884c7b2c..496f30c9740 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -205,7 +205,25 @@ function decorateLegacyGraph( // Step 3: Add bundles to their bundle groups for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); - let entryBundle = nullthrows(idealBundleGraph.getNode(bundleId)); + for (let id of outboundNodeIds) { + let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); + invariant(siblingBundle !== 'root'); + let legacySiblingBundle = nullthrows( + idealBundleToLegacyBundle.get(siblingBundle), + ); + bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); + } + } + + idealBundleGraph.traverse((nodeId, _, actions) => { + let node = idealBundleGraph.getNode(nodeId); + if (node === 'root') { + return; + } + actions.skipChildren(); + + let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(nodeId); + let entryBundle = nullthrows(idealBundleGraph.getNode(nodeId)); invariant(entryBundle !== 'root'); let legacyEntryBundle = nullthrows( idealBundleToLegacyBundle.get(entryBundle), @@ -216,10 +234,9 @@ function decorateLegacyGraph( let legacySiblingBundle = nullthrows( idealBundleToLegacyBundle.get(siblingBundle), ); - bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); bundleGraph.createBundleReference(legacyEntryBundle, legacySiblingBundle); } - } + }); // Step 4: Add references to all bundles for (let [asset, references] of idealGraph.assetReference) { @@ -568,11 +585,27 @@ function createIdealGraph( root.type !== node.value.type) ) { if (!isAsync) { - bundleGraph.addEdge( + // bundleGraph.addEdge( + // nullthrows(bundles.get(root.id)), + // nullthrows(bundles.get(node.value.id)), + // ); + let bundleGroupIds = bundleGraph.getNodeIdsConnectedTo( nullthrows(bundles.get(root.id)), - nullthrows(bundles.get(node.value.id)), ); + for (let bundleGroupId of bundleGroupIds) { + bundleGraph.addEdge( + bundleGroupId, + nullthrows(bundles.get(node.value.id)), + ); + } } + console.log( + 'skipping', + node.value.filePath, + 'from', + root.filePath, + isAsync, + ); actions.skipChildren(); return; } From a4744138bbe65070d134f609ca9442e1b46dbb4c Mon Sep 17 00:00:00 2001 From: thebriando Date: Tue, 15 Feb 2022 15:36:47 -0800 Subject: [PATCH 08/87] * Get inline bundles in bundle group in HTML packager * Traverse each bundle instead of iterating each outbound node * Add edge between root and bundle --- .../experimental/src/ExperimentalBundler.js | 57 ++++++++++--------- packages/core/core/src/BundleGraph.js | 11 ++-- packages/packagers/html/src/HTMLPackager.js | 15 +++-- 3 files changed, 45 insertions(+), 38 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 496f30c9740..61bb1249f32 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -204,15 +204,20 @@ function decorateLegacyGraph( // Step 3: Add bundles to their bundle groups for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { - let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(bundleId); - for (let id of outboundNodeIds) { - let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); - invariant(siblingBundle !== 'root'); - let legacySiblingBundle = nullthrows( - idealBundleToLegacyBundle.get(siblingBundle), - ); - bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); - } + idealBundleGraph.traverse( + { + exit: (nodeId, _, actions) => { + let siblingBundle = nullthrows(idealBundleGraph.getNode(nodeId)); + invariant(siblingBundle !== 'root'); + let legacySiblingBundle = nullthrows( + idealBundleToLegacyBundle.get(siblingBundle), + ); + + bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); + }, + }, + bundleId, + ); } idealBundleGraph.traverse((nodeId, _, actions) => { @@ -585,27 +590,11 @@ function createIdealGraph( root.type !== node.value.type) ) { if (!isAsync) { - // bundleGraph.addEdge( - // nullthrows(bundles.get(root.id)), - // nullthrows(bundles.get(node.value.id)), - // ); - let bundleGroupIds = bundleGraph.getNodeIdsConnectedTo( + bundleGraph.addEdge( nullthrows(bundles.get(root.id)), + nullthrows(bundles.get(node.value.id)), ); - for (let bundleGroupId of bundleGroupIds) { - bundleGraph.addEdge( - bundleGroupId, - nullthrows(bundles.get(node.value.id)), - ); - } } - console.log( - 'skipping', - node.value.filePath, - 'from', - root.filePath, - isAsync, - ); actions.skipChildren(); return; } @@ -716,6 +705,20 @@ function createIdealGraph( } } } + for (let bundleNodeId of bundleGroupBundleIds) { + let bundleNode = nullthrows(bundleGraph.getNode(bundleNodeId)); + if ( + bundleNode.bundleBehavior !== 'isolated' && + bundleNode.bundleBehavior !== 'inline' + ) { + continue; + } + bundleGraph.traverse((nodeId, _, actions) => { + let node = nullthrows(bundleGraph.getNode(nodeId)); + invariant(node !== 'root'); + ancestorAssets.set([...node.assets][0], new Map()); + }, bundleNodeId); + } // Step 5: Place all assets into bundles or create shared bundles. Each asset // is placed into a single bundle based on the bundle entries it is reachable from. diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 537ecc0f78a..dd55029dba0 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -1243,6 +1243,7 @@ export default class BundleGraph { bundleGroup: BundleGroup, opts?: {|includeInline: boolean|}, ): Array { + debugger; let bundles: Set = new Set(); for (let bundleNodeId of this._graph.getNodeIdsConnectedFrom( this._graph.getNodeIdByContentKey(getBundleGroupId(bundleGroup)), @@ -1258,11 +1259,11 @@ export default class BundleGraph { bundles.add(bundle); } - for (let referencedBundle of this.getReferencedBundles(bundle, { - includeInline: true, - })) { - bundles.add(referencedBundle); - } + // for (let referencedBundle of this.getReferencedBundles(bundle, { + // includeInline: true, + // })) { + // bundles.add(referencedBundle); + // } } return [...bundles]; diff --git a/packages/packagers/html/src/HTMLPackager.js b/packages/packagers/html/src/HTMLPackager.js index b30faf3a110..5c5994780de 100644 --- a/packages/packagers/html/src/HTMLPackager.js +++ b/packages/packagers/html/src/HTMLPackager.js @@ -51,12 +51,15 @@ export default (new Packager({ // Add bundles in the same bundle group that are not inline. For example, if two inline // bundles refer to the same library that is extracted into a shared bundle. - let referencedBundles = [ - ...setDifference( - new Set(bundleGraph.getReferencedBundles(bundle)), - new Set(bundleGraph.getReferencedBundles(bundle, {recursive: false})), - ), - ]; + let referencedBundles = bundleGraph + .getBundlesInBundleGroup( + nullthrows( + bundleGraph + .getBundleGroupsContainingBundle(bundle) + .find(b => b.entryAssetId === nullthrows(bundle.getMainEntry()).id), + ), + ) + .filter(b => b.bundleBehavior !== 'inline' && b.id !== bundle.id); let renderConfig = config?.render; let {html} = await posthtml([ From 0a9f32df901859871f5f0b3aa1671df44440fe04 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 16 Feb 2022 13:00:10 -0800 Subject: [PATCH 09/87] use and follow reference edges again --- .../experimental/src/ExperimentalBundler.js | 20 ++----------------- packages/core/core/src/BundleGraph.js | 11 +++++----- packages/packagers/html/src/HTMLPackager.js | 16 +++++++-------- 3 files changed, 14 insertions(+), 33 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 61bb1249f32..e6035075f23 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -203,23 +203,6 @@ function decorateLegacyGraph( } // Step 3: Add bundles to their bundle groups - for (let [bundleId, bundleGroup] of entryBundleToBundleGroup) { - idealBundleGraph.traverse( - { - exit: (nodeId, _, actions) => { - let siblingBundle = nullthrows(idealBundleGraph.getNode(nodeId)); - invariant(siblingBundle !== 'root'); - let legacySiblingBundle = nullthrows( - idealBundleToLegacyBundle.get(siblingBundle), - ); - - bundleGraph.addBundleToBundleGroup(legacySiblingBundle, bundleGroup); - }, - }, - bundleId, - ); - } - idealBundleGraph.traverse((nodeId, _, actions) => { let node = idealBundleGraph.getNode(nodeId); if (node === 'root') { @@ -233,6 +216,7 @@ function decorateLegacyGraph( let legacyEntryBundle = nullthrows( idealBundleToLegacyBundle.get(entryBundle), ); + for (let id of outboundNodeIds) { let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); invariant(siblingBundle !== 'root'); @@ -713,7 +697,7 @@ function createIdealGraph( ) { continue; } - bundleGraph.traverse((nodeId, _, actions) => { + bundleGraph.traverse(nodeId => { let node = nullthrows(bundleGraph.getNode(nodeId)); invariant(node !== 'root'); ancestorAssets.set([...node.assets][0], new Map()); diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index dd55029dba0..537ecc0f78a 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -1243,7 +1243,6 @@ export default class BundleGraph { bundleGroup: BundleGroup, opts?: {|includeInline: boolean|}, ): Array { - debugger; let bundles: Set = new Set(); for (let bundleNodeId of this._graph.getNodeIdsConnectedFrom( this._graph.getNodeIdByContentKey(getBundleGroupId(bundleGroup)), @@ -1259,11 +1258,11 @@ export default class BundleGraph { bundles.add(bundle); } - // for (let referencedBundle of this.getReferencedBundles(bundle, { - // includeInline: true, - // })) { - // bundles.add(referencedBundle); - // } + for (let referencedBundle of this.getReferencedBundles(bundle, { + includeInline: true, + })) { + bundles.add(referencedBundle); + } } return [...bundles]; diff --git a/packages/packagers/html/src/HTMLPackager.js b/packages/packagers/html/src/HTMLPackager.js index 5c5994780de..dfc7057bcb5 100644 --- a/packages/packagers/html/src/HTMLPackager.js +++ b/packages/packagers/html/src/HTMLPackager.js @@ -51,15 +51,13 @@ export default (new Packager({ // Add bundles in the same bundle group that are not inline. For example, if two inline // bundles refer to the same library that is extracted into a shared bundle. - let referencedBundles = bundleGraph - .getBundlesInBundleGroup( - nullthrows( - bundleGraph - .getBundleGroupsContainingBundle(bundle) - .find(b => b.entryAssetId === nullthrows(bundle.getMainEntry()).id), - ), - ) - .filter(b => b.bundleBehavior !== 'inline' && b.id !== bundle.id); + let referencedBundles = [ + ...setDifference( + new Set(bundleGraph.getReferencedBundles(bundle)), + new Set(bundleGraph.getReferencedBundles(bundle, {recursive: false})), + ), + ]; + let renderConfig = config?.render; let {html} = await posthtml([ From 73ad8286f58bdd1e8a036665b439fb65697df67b Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Thu, 17 Feb 2022 13:26:14 -0800 Subject: [PATCH 10/87] set env whenever we create bundles --- packages/bundlers/experimental/src/ExperimentalBundler.js | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e6035075f23..1d20bb4a55b 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -148,6 +148,7 @@ function decorateLegacyGraph( needsStableName: idealBundle.needsStableName, bundleBehavior: idealBundle.bundleBehavior, target: idealBundle.target, + env: idealBundle.env, }), ); @@ -172,6 +173,7 @@ function decorateLegacyGraph( needsStableName: idealBundle.needsStableName, bundleBehavior: idealBundle.bundleBehavior, target: idealBundle.target, + env: idealBundle.env, }), ); } @@ -370,6 +372,7 @@ function createIdealGraph( invariant(firstBundleGroup !== 'root'); bundle = createBundle({ asset: childAsset, + env: firstBundleGroup.target.env, target: firstBundleGroup.target, needsStableName: dependency.bundleBehavior === 'inline' || @@ -460,6 +463,7 @@ function createIdealGraph( // Create a new bundle if none of the same type exists already. bundle = createBundle({ asset: childAsset, + env: bundleGroup.target.env, target: bundleGroup.target, needsStableName: childAsset.bundleBehavior === 'inline' || From eb3703bab99a52607af74f7c1b9dc24bf86f2159 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Fri, 18 Feb 2022 18:15:13 -0800 Subject: [PATCH 11/87] Check to add parallel edges from all paths to an asset from a bundle --- .../experimental/src/ExperimentalBundler.js | 39 +++++++++++-------- packages/core/core/src/BundleGraph.js | 2 +- 2 files changed, 24 insertions(+), 17 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 1d20bb4a55b..ddfdaaf49c4 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -535,7 +535,7 @@ function createIdealGraph( let reachableRoots: ContentGraph = new ContentGraph(); for (let [root] of bundleRoots) { let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); - assetGraph.traverse((node, isAsync, actions) => { + assetGraph.traverse((node, _, actions) => { if (node.value === root) { return; } @@ -547,7 +547,7 @@ function createIdealGraph( if (dependency.priority !== 'sync') { let assets = assetGraph.getDependencyAssets(dependency); if (assets.length === 0) { - return true; + return; } invariant(assets.length === 1); @@ -568,22 +568,29 @@ function createIdealGraph( } } } - return dependency.priority !== 'sync'; - } - if ( - bundleRoots.has(node.value) && - ((root.isBundleSplittable && !entries.has(root)) || - isAsync || - root.type !== node.value.type) - ) { - if (!isAsync) { - bundleGraph.addEdge( - nullthrows(bundles.get(root.id)), - nullthrows(bundles.get(node.value.id)), - ); + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return; } - actions.skipChildren(); + + invariant(assets.length === 1); + let resolved = assets[0]; + let isAsync = dependency.priority !== 'sync'; + if ( + bundleRoots.has(resolved) && + ((root.isBundleSplittable && !entries.has(root)) || + isAsync || + root.type !== resolved.type) + ) { + let rootNodeId = nullthrows(bundles.get(root.id)); + let resolvedNodeId = nullthrows(bundles.get(resolved.id)); + if (!isAsync && !bundleGraph.hasEdge(rootNodeId, resolvedNodeId)) { + bundleGraph.addEdge(rootNodeId, resolvedNodeId); + } + actions.skipChildren(); + } + return; } diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 537ecc0f78a..03f2fba5b36 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -1399,7 +1399,7 @@ export default class BundleGraph { for (let dep of deps) { let depSymbols = dep.symbols; if (!depSymbols) { - found = true; + // found = true; continue; } // If this is a re-export, find the original module. From 463bed090e25fa03a98e53fb1b804877e5148e52 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 22 Feb 2022 13:32:41 -0800 Subject: [PATCH 12/87] Always register referenced bundles before pruning when building bundle manifest --- packages/runtimes/js/src/JSRuntime.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/runtimes/js/src/JSRuntime.js b/packages/runtimes/js/src/JSRuntime.js index f7751dbe971..a05f1cf7a13 100644 --- a/packages/runtimes/js/src/JSRuntime.js +++ b/packages/runtimes/js/src/JSRuntime.js @@ -552,6 +552,11 @@ function getRegisterCode( idToName[bundle.publicId] = path.basename(nullthrows(bundle.name)); if (bundle !== entryBundle && isNewContext(bundle, bundleGraph)) { + for (let referenced of bundleGraph.getReferencedBundles(bundle)) { + idToName[referenced.publicId] = path.basename( + nullthrows(referenced.name), + ); + } // New contexts have their own manifests, so there's no need to continue. actions.skipChildren(); } From dbfab75f5256ca3399255d7f18f5d3626549aa11 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 22 Feb 2022 16:10:27 -0800 Subject: [PATCH 13/87] Revert "set env whenever we create bundles" This reverts commit 73ad8286f58bdd1e8a036665b439fb65697df67b. --- packages/bundlers/experimental/src/ExperimentalBundler.js | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index ddfdaaf49c4..99a7e0be135 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -148,7 +148,6 @@ function decorateLegacyGraph( needsStableName: idealBundle.needsStableName, bundleBehavior: idealBundle.bundleBehavior, target: idealBundle.target, - env: idealBundle.env, }), ); @@ -173,7 +172,6 @@ function decorateLegacyGraph( needsStableName: idealBundle.needsStableName, bundleBehavior: idealBundle.bundleBehavior, target: idealBundle.target, - env: idealBundle.env, }), ); } @@ -372,7 +370,6 @@ function createIdealGraph( invariant(firstBundleGroup !== 'root'); bundle = createBundle({ asset: childAsset, - env: firstBundleGroup.target.env, target: firstBundleGroup.target, needsStableName: dependency.bundleBehavior === 'inline' || @@ -463,7 +460,6 @@ function createIdealGraph( // Create a new bundle if none of the same type exists already. bundle = createBundle({ asset: childAsset, - env: bundleGroup.target.env, target: bundleGroup.target, needsStableName: childAsset.bundleBehavior === 'inline' || From 757eff848690427a3859cf0ac21069c3d4b4562b Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 22 Feb 2022 16:55:52 -0800 Subject: [PATCH 14/87] Add test for referenced roots in bundle manifest --- packages/core/integration-tests/test/html.js | 16 ++++++++++++++++ .../integration/html-shared-referenced/async.js | 3 +++ .../integration/html-shared-referenced/async2.js | 0 .../html-shared-referenced/index1.html | 1 + .../integration/html-shared-referenced/index1.js | 1 + .../html-shared-referenced/index2.html | 1 + .../integration/html-shared-referenced/index2.js | 3 +++ .../html-shared-referenced/package.json | 5 +++++ .../integration/html-shared-referenced/shared.js | 1 + .../integration/html-shared-referenced/yarn.lock | 0 10 files changed, 31 insertions(+) create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/async.js create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/async2.js create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/index1.html create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/index1.js create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/index2.html create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/index2.js create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/package.json create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/shared.js create mode 100644 packages/core/integration-tests/test/integration/html-shared-referenced/yarn.lock diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index fd42fc4006d..55061b33568 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -2835,4 +2835,20 @@ describe('html', function () { }, ); }); + + it('extracts shared bundles that load referenced bundle roots across entries', async () => { + let b = await bundle( + ['index1.html', 'index2.html'].map(entry => + path.join(__dirname, 'integration/html-shared-referenced', entry), + ), + { + mode: 'production', + defaultTargetOptions: { + shouldOptimize: false, + }, + }, + ); + + await run(b); + }); }); diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/async.js b/packages/core/integration-tests/test/integration/html-shared-referenced/async.js new file mode 100644 index 00000000000..ad07c1e9102 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-shared-referenced/async.js @@ -0,0 +1,3 @@ +import './async2.js'; + +import('./async2.js'); diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/async2.js b/packages/core/integration-tests/test/integration/html-shared-referenced/async2.js new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/index1.html b/packages/core/integration-tests/test/integration/html-shared-referenced/index1.html new file mode 100644 index 00000000000..befa6f91239 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-shared-referenced/index1.html @@ -0,0 +1 @@ + diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/index1.js b/packages/core/integration-tests/test/integration/html-shared-referenced/index1.js new file mode 100644 index 00000000000..c21c06aca8a --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-shared-referenced/index1.js @@ -0,0 +1 @@ +import './shared'; diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/index2.html b/packages/core/integration-tests/test/integration/html-shared-referenced/index2.html new file mode 100644 index 00000000000..edbd81997e9 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-shared-referenced/index2.html @@ -0,0 +1 @@ + diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/index2.js b/packages/core/integration-tests/test/integration/html-shared-referenced/index2.js new file mode 100644 index 00000000000..04c9991d158 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-shared-referenced/index2.js @@ -0,0 +1,3 @@ +import './async.js'; +import './shared.js'; + diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/package.json b/packages/core/integration-tests/test/integration/html-shared-referenced/package.json new file mode 100644 index 00000000000..c5f216e03fd --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-shared-referenced/package.json @@ -0,0 +1,5 @@ +{ + "@parcel/bundler-default": { + "minBundleSize": 0 + } +} diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/shared.js b/packages/core/integration-tests/test/integration/html-shared-referenced/shared.js new file mode 100644 index 00000000000..6c111b7bca1 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-shared-referenced/shared.js @@ -0,0 +1 @@ +import('./async.js'); diff --git a/packages/core/integration-tests/test/integration/html-shared-referenced/yarn.lock b/packages/core/integration-tests/test/integration/html-shared-referenced/yarn.lock new file mode 100644 index 00000000000..e69de29bb2d From 6e22065de143f5fe0a5f13842f728ce462463e9b Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Mon, 28 Feb 2022 14:33:36 -0800 Subject: [PATCH 15/87] Add reused sibling bundles to asyncBundleRootGraph --- .../experimental/src/ExperimentalBundler.js | 44 ++++++++++++------- 1 file changed, 28 insertions(+), 16 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 99a7e0be135..652e545aa25 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -581,8 +581,34 @@ function createIdealGraph( ) { let rootNodeId = nullthrows(bundles.get(root.id)); let resolvedNodeId = nullthrows(bundles.get(resolved.id)); - if (!isAsync && !bundleGraph.hasEdge(rootNodeId, resolvedNodeId)) { - bundleGraph.addEdge(rootNodeId, resolvedNodeId); + if (!isAsync) { + if (!bundleGraph.hasEdge(rootNodeId, resolvedNodeId)) { + bundleGraph.addEdge(rootNodeId, resolvedNodeId); + } + + // Reflect this connection in the async bundle root graph by + // connecting the reused bundle to every case where the original + // root bundle is loaded. This only necessary in cases that + // bundles in a group are executed in serial (e.g. js referenced + // by html) + for (let inboundAsync of asyncBundleRootGraph.getNodeIdsConnectedTo( + nullthrows(asyncBundleRootGraph.getNodeIdByContentKey(root.id)), + )) { + let resolvedInAsyncRootGraph = nullthrows( + asyncBundleRootGraph.getNodeIdByContentKey(resolved.id), + ); + if ( + !asyncBundleRootGraph.hasEdge( + inboundAsync, + resolvedInAsyncRootGraph, + ) + ) { + asyncBundleRootGraph.addEdge( + inboundAsync, + resolvedInAsyncRootGraph, + ); + } + } } actions.skipChildren(); } @@ -696,20 +722,6 @@ function createIdealGraph( } } } - for (let bundleNodeId of bundleGroupBundleIds) { - let bundleNode = nullthrows(bundleGraph.getNode(bundleNodeId)); - if ( - bundleNode.bundleBehavior !== 'isolated' && - bundleNode.bundleBehavior !== 'inline' - ) { - continue; - } - bundleGraph.traverse(nodeId => { - let node = nullthrows(bundleGraph.getNode(nodeId)); - invariant(node !== 'root'); - ancestorAssets.set([...node.assets][0], new Map()); - }, bundleNodeId); - } // Step 5: Place all assets into bundles or create shared bundles. Each asset // is placed into a single bundle based on the bundle entries it is reachable from. From f347cac94333b9592c0280773058472a98b2453e Mon Sep 17 00:00:00 2001 From: thebriando Date: Tue, 1 Mar 2022 13:44:51 -0800 Subject: [PATCH 16/87] Add test case for asset that has both an async and sync import --- packages/core/integration-tests/test/html.js | 14 ++++++++++++++ .../integration/html-sync-async-asset/index.html | 1 + .../integration/html-sync-async-asset/index.js | 4 ++++ .../integration/html-sync-async-asset/other.js | 1 + .../test/integration/html-sync-async-asset/test.js | 1 + 5 files changed, 21 insertions(+) create mode 100644 packages/core/integration-tests/test/integration/html-sync-async-asset/index.html create mode 100644 packages/core/integration-tests/test/integration/html-sync-async-asset/index.js create mode 100644 packages/core/integration-tests/test/integration/html-sync-async-asset/other.js create mode 100644 packages/core/integration-tests/test/integration/html-sync-async-asset/test.js diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index 55061b33568..6500079ed63 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -2851,4 +2851,18 @@ describe('html', function () { await run(b); }); + + it('should not skip bundleRoots if an asset is both async required and static required', async function () { + let b = await bundle( + path.join(__dirname, 'integration/html-sync-async-asset/index.html'), + { + mode: 'production', + defaultTargetOptions: { + shouldOptimize: false, + }, + }, + ); + + await run(b, {output: null}, {require: false}); + }); }); diff --git a/packages/core/integration-tests/test/integration/html-sync-async-asset/index.html b/packages/core/integration-tests/test/integration/html-sync-async-asset/index.html new file mode 100644 index 00000000000..8ef9a70b216 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-sync-async-asset/index.html @@ -0,0 +1 @@ + diff --git a/packages/core/integration-tests/test/integration/html-sync-async-asset/index.js b/packages/core/integration-tests/test/integration/html-sync-async-asset/index.js new file mode 100644 index 00000000000..afd241a43f8 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-sync-async-asset/index.js @@ -0,0 +1,4 @@ +import t from "./test.js"; + +import("./other.js") + .then((v) => v.default) diff --git a/packages/core/integration-tests/test/integration/html-sync-async-asset/other.js b/packages/core/integration-tests/test/integration/html-sync-async-asset/other.js new file mode 100644 index 00000000000..6be91ea6cc7 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-sync-async-asset/other.js @@ -0,0 +1 @@ +export default import("./test.js"); diff --git a/packages/core/integration-tests/test/integration/html-sync-async-asset/test.js b/packages/core/integration-tests/test/integration/html-sync-async-asset/test.js new file mode 100644 index 00000000000..58c57157d36 --- /dev/null +++ b/packages/core/integration-tests/test/integration/html-sync-async-asset/test.js @@ -0,0 +1 @@ +export default "test"; From a4162b2729807d90f9fd02ae81e2d8249e9d9366 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 1 Mar 2022 13:52:14 -0800 Subject: [PATCH 17/87] ExperimentalBundler: stop at isolated bundles --- packages/bundlers/experimental/src/ExperimentalBundler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 652e545aa25..cfbbb463516 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -577,11 +577,12 @@ function createIdealGraph( bundleRoots.has(resolved) && ((root.isBundleSplittable && !entries.has(root)) || isAsync || + resolved.bundleBehavior === 'isolated' || root.type !== resolved.type) ) { let rootNodeId = nullthrows(bundles.get(root.id)); let resolvedNodeId = nullthrows(bundles.get(resolved.id)); - if (!isAsync) { + if (!isAsync && resolved.bundleBehavior !== 'isolated') { if (!bundleGraph.hasEdge(rootNodeId, resolvedNodeId)) { bundleGraph.addEdge(rootNodeId, resolvedNodeId); } From 2a026313c4d9b4c92df1ab1027504133f2e39ca9 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 1 Mar 2022 14:40:23 -0800 Subject: [PATCH 18/87] ExperimentalBundler: fix step 7 comment --- packages/bundlers/experimental/src/ExperimentalBundler.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index cfbbb463516..e59a36bc588 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -879,8 +879,8 @@ function createIdealGraph( } } - // Step 7: Merge any sibling bundles required by entry bundles back into the entry bundle. - // Entry bundles must be predictable, so cannot have unpredictable siblings. + // Step 7: Merge any shared bundles under the minimum bundle size back into + // their source bundles, and remove the bundle. for (let [bundleNodeId, bundle] of bundleGraph.nodes) { if (bundle === 'root') continue; if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { From 0d20d02cebded3abfe943fde15da45906397c74b Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 1 Mar 2022 15:10:39 -0800 Subject: [PATCH 19/87] ExperimentalBundler: initialize entry bundles with no ancestors --- packages/bundlers/experimental/src/ExperimentalBundler.js | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e59a36bc588..5bce17b7306 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -639,6 +639,11 @@ function createIdealGraph( > = new DefaultMap(() => new DefaultMap(() => 0)); // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group + for (let entry of entries.keys()) { + // Initialize an empty set of ancestors available to entries + ancestorAssets.set(entry, new Map()); + } + for (let nodeId of asyncBundleRootGraph.topoSort()) { const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; From 22ba3bb2bcc9d1b636254f9eca660bcb599d04b3 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 1 Mar 2022 15:10:59 -0800 Subject: [PATCH 20/87] ExperimentalBundler: accept shared bundles extracted from workers --- .../core/integration-tests/test/javascript.js | 46 +++++++++++++------ 1 file changed, 32 insertions(+), 14 deletions(-) diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index dda211e5016..c53f2a6d1fd 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1019,25 +1019,43 @@ describe('javascript', function () { }, ); - assertBundles(b, [ - { - assets: ['dedicated-worker.js', 'index.js'], - }, - { - name: 'index.js', - assets: ['index.js', 'bundle-url.js', 'get-worker-url.js'], - }, - { - assets: ['shared-worker.js', 'index.js'], - }, - ]); + if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { + assertBundles(b, [ + { + assets: ['dedicated-worker.js'], + }, + { + name: 'index.js', + assets: ['index.js', 'bundle-url.js', 'get-worker-url.js'], + }, + { + assets: ['shared-worker.js'], + }, + { + assets: ['index.js'], + }, + ]); + } else { + assertBundles(b, [ + { + assets: ['dedicated-worker.js', 'index.js'], + }, + { + name: 'index.js', + assets: ['index.js', 'bundle-url.js', 'get-worker-url.js'], + }, + { + assets: ['shared-worker.js', 'index.js'], + }, + ]); + } let dedicated, shared; b.traverseBundles((bundle, ctx, traversal) => { - if (bundle.getMainEntry().filePath.endsWith('shared-worker.js')) { + if (bundle.getMainEntry()?.filePath.endsWith('shared-worker.js')) { shared = bundle; } else if ( - bundle.getMainEntry().filePath.endsWith('dedicated-worker.js') + bundle.getMainEntry()?.filePath.endsWith('dedicated-worker.js') ) { dedicated = bundle; } From df5a741659a6a75e0a5a584a7181c900f1a23351 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 1 Mar 2022 17:53:11 -0800 Subject: [PATCH 21/87] Remove unused async bundles if needed --- .../experimental/src/ExperimentalBundler.js | 52 +++++++++++++------ 1 file changed, 36 insertions(+), 16 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 5bce17b7306..6e94214b317 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -792,29 +792,26 @@ function createIdealGraph( } } - let willInternalizeRoots = asyncBundleRootGraph - .getNodeIdsConnectedTo( - asyncBundleRootGraph.getNodeIdByContentKey(asset.id), - ) + let asyncAssetId = asyncBundleRootGraph.getNodeIdByContentKey(asset.id); + let loadedBy = asyncBundleRootGraph + .getNodeIdsConnectedTo(asyncAssetId) .map(id => nullthrows(asyncBundleRootGraph.getNode(id))) - .filter(bundleRoot => { - if (bundleRoot === 'root') { - return false; - } - - return ( - reachableRoots.hasEdge( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - reachableRoots.getNodeIdByContentKey(asset.id), - ) || ancestorAssets.get(bundleRoot)?.has(asset) - ); - }) + .filter(bundleRoot => bundleRoot !== 'root') .map(bundleRoot => { // For Flow invariant(bundleRoot !== 'root'); return bundleRoot; }); + let reachableAssetId = reachableRoots.getNodeIdByContentKey(asset.id); + let willInternalizeRoots = loadedBy.filter( + bundleRoot => + reachableRoots.hasEdge( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + reachableAssetId, + ) || ancestorAssets.get(bundleRoot)?.has(asset), + ); + for (let bundleRoot of willInternalizeRoots) { if (bundleRoot !== asset) { let bundle = nullthrows( @@ -822,10 +819,33 @@ function createIdealGraph( ); invariant(bundle !== 'root'); bundle.internalizedAssetIds.push(asset.id); + asyncBundleRootGraph.removeEdge( + asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + asyncAssetId, + ); } } + + if ( + !entries.has(asset) && + loadedBy.length > 0 && + loadedBy.length === willInternalizeRoots.length + ) { + // The contents of this async bundle are accessible already in every + // use without it. Remove the async bundle entirely. + let bundleId = nullthrows(bundles.get(asset.id)); + bundleGraph.removeNode(bundleId); + bundles.delete(asset.id); + bundleRoots.delete(asset); + if (asyncBundleRootGraph.hasNode(asyncAssetId)) { + asyncBundleRootGraph.removeNode(asyncAssetId); + } + ancestorAssets.delete(asset); + reachableRoots.replaceNodeIdsConnectedTo(reachableAssetId, []); + } } } + if (reachable.length > 0) { let reachableEntries = reachable.filter( a => entries.has(a) || !a.isBundleSplittable, From 731c07b868098c54797e6c61543d1a60b72b9abb Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 2 Mar 2022 12:06:29 -0800 Subject: [PATCH 22/87] Scope-hositing with new bundler: allow less duplication --- packages/core/integration-tests/test/scope-hoisting.js | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/packages/core/integration-tests/test/scope-hoisting.js b/packages/core/integration-tests/test/scope-hoisting.js index b1632bfc83e..7104088f303 100644 --- a/packages/core/integration-tests/test/scope-hoisting.js +++ b/packages/core/integration-tests/test/scope-hoisting.js @@ -5008,7 +5008,11 @@ describe('scope hoisting', function () { ], }, {assets: ['dep.js']}, - {assets: ['async-has-dep.js', 'dep.js', 'get-dep.js']}, + { + assets: process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER + ? ['async-has-dep.js'] + : ['async-has-dep.js', 'dep.js', 'get-dep.js'], + }, {assets: ['get-dep.js']}, ]); From 345e0cc3a7670c27780382656a8a21c89b7472ec Mon Sep 17 00:00:00 2001 From: thebriando Date: Wed, 2 Mar 2022 23:51:38 -0800 Subject: [PATCH 23/87] Uncomment line in getSymbolResolution --- packages/core/core/src/BundleGraph.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 03f2fba5b36..537ecc0f78a 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -1399,7 +1399,7 @@ export default class BundleGraph { for (let dep of deps) { let depSymbols = dep.symbols; if (!depSymbols) { - // found = true; + found = true; continue; } // If this is a re-export, find the original module. From 727dde71309ffe564fb738641cc247cb24feb360 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Thu, 3 Mar 2022 13:07:24 -0800 Subject: [PATCH 24/87] Consider sibling availability before removing from ancestorAssets --- .../experimental/src/ExperimentalBundler.js | 86 +++++++++---------- 1 file changed, 41 insertions(+), 45 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 6e94214b317..b4d370affa6 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -627,10 +627,14 @@ function createIdealGraph( // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets - let ancestorAssets: Map< + let ancestorAssets: Map> = new Map(); + let siblingAssets: DefaultMap< BundleRoot, - Map | null>, - > = new Map(); + Map, + > = new DefaultMap(() => new Map()); + let borrowedAssets: DefaultMap> = new DefaultMap( + () => new Set(), + ); // Reference count of each asset available within a given bundleRoot's bundle group let assetRefsInBundleGroup: DefaultMap< @@ -641,21 +645,18 @@ function createIdealGraph( // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group for (let entry of entries.keys()) { // Initialize an empty set of ancestors available to entries - ancestorAssets.set(entry, new Map()); + ancestorAssets.set(entry, new Set()); } for (let nodeId of asyncBundleRootGraph.topoSort()) { const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); - let ancestors = ancestorAssets.get(bundleRoot); - // First consider bundle group asset availability, processing only // non-isolated bundles within that bundle group let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; - // Map of assets in the bundle group to their refcounts - let assetRefs = assetRefsInBundleGroup.get(bundleRoot); + let available = new Set(ancestorAssets.get(bundleRoot)); for (let bundleIdInGroup of [ bundleGroupId, ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), @@ -668,63 +669,58 @@ function createIdealGraph( ) { continue; } - let [bundleRoot] = [...bundleInGroup.assets]; + let [siblingBundleRoot] = [...bundleInGroup.assets]; // Assets directly connected to current bundleRoot let assetsFromBundleRoot = reachableRoots .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), + reachableRoots.getNodeIdByContentKey(siblingBundleRoot.id), ) .map(id => nullthrows(reachableRoots.getNode(id))); - for (let asset of [bundleRoot, ...assetsFromBundleRoot]) { - assetRefs.set(asset, assetRefs.get(asset) + 1); + for (let asset of [siblingBundleRoot, ...assetsFromBundleRoot]) { + available.add(asset); + if (bundleIdInGroup !== bundleGroupId) { + siblingAssets.get(bundleRoot).set(asset, siblingBundleRoot); + } } } - // Enumerate bundleRoots connected to the node (parent), taking the intersection - // between the assets synchronously loaded by the parent, and those loaded by the child. - let bundleGroupAssets = new Set(assetRefs.keys()); - - let combined = ancestors - ? new Map([...bundleGroupAssets, ...ancestors.keys()].map(a => [a, null])) - : new Map([...bundleGroupAssets].map(a => [a, null])); - + let childrenAssets: DefaultMap> = new DefaultMap( + () => [], + ); let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); + // Group assets available across our children by the child. This will be used + // to determine borrowers if needed below. for (let childId of children) { let child = asyncBundleRootGraph.getNode(childId); invariant(child !== 'root' && child != null); - const availableAssets = ancestorAssets.get(child); - if (availableAssets != null) { - ancestryIntersect(availableAssets, combined); - } else { - ancestorAssets.set(child, combined); + let assets = reachableRoots + .getNodeIdsConnectedFrom(reachableRoots.getNodeIdByContentKey(child.id)) + .map(id => nullthrows(reachableRoots.getNode(id))); + for (let asset of [child, ...assets]) { + childrenAssets.get(asset).push(child); } } - let siblingAncestors = ancestors - ? ancestryUnion(new Set(ancestors.keys()), assetRefs, bundleRoot) - : new Map([...bundleGroupAssets].map(a => [a, [bundleRoot]])); - - for (let bundleIdInGroup of bundleGraph.getNodeIdsConnectedFrom( - bundleGroupId, - )) { - let bundleInGroup = bundleGraph.getNode(bundleIdInGroup); - invariant( - bundleInGroup != null && - bundleInGroup !== 'root' && - bundleInGroup.assets != null, - ); - - let [bundleRoot] = [...bundleInGroup.assets]; - - const availableAssets = ancestorAssets.get(bundleRoot); + for (let childId of children) { + let child = asyncBundleRootGraph.getNode(childId); + invariant(child !== 'root' && child != null); + const childAvailableAssets = ancestorAssets.get(child); - if (availableAssets != null) { - ancestryIntersect(availableAssets, siblingAncestors); + if (childAvailableAssets != null) { + for (let asset of childAvailableAssets) { + if (!available.has(asset)) { + if (childrenAssets.has(asset)) { + borrowedAssets.get(childrenAssets.get(asset)[0]).add(asset); + } else { + childAvailableAssets.delete(asset); + } + } + } } else { - ancestorAssets.set(bundleRoot, siblingAncestors); + ancestorAssets.set(child, new Set(available)); } } } From 2032542f3960fbb18c304a9c6a102e4ceefacaa3 Mon Sep 17 00:00:00 2001 From: thebriando Date: Wed, 2 Mar 2022 23:51:38 -0800 Subject: [PATCH 25/87] Uncomment line in getSymbolResolution --- packages/core/core/src/BundleGraph.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 03f2fba5b36..537ecc0f78a 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -1399,7 +1399,7 @@ export default class BundleGraph { for (let dep of deps) { let depSymbols = dep.symbols; if (!depSymbols) { - // found = true; + found = true; continue; } // If this is a re-export, find the original module. From 37e5c6a900a61ef884c5a95d25a06beeb93315d5 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 8 Mar 2022 15:28:31 -0800 Subject: [PATCH 26/87] Upgrade flow to 0.173.0 (#7809) --- .flowconfig | 2 +- package.json | 2 +- yarn.lock | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.flowconfig b/.flowconfig index ce0fd3823b0..61557139c3d 100644 --- a/.flowconfig +++ b/.flowconfig @@ -36,4 +36,4 @@ untyped-import untyped-type-import [version] -0.171.0 +0.173.0 diff --git a/package.json b/package.json index c75f576aca0..83eec65c1eb 100644 --- a/package.json +++ b/package.json @@ -42,7 +42,7 @@ "@types/node": "^15.12.4", "cross-env": "^7.0.0", "eslint": "^7.20.0", - "flow-bin": "0.171.0", + "flow-bin": "0.173.0", "glob": "^7.1.6", "gulp": "^4.0.2", "gulp-babel": "^8.0.0", diff --git a/yarn.lock b/yarn.lock index 66a27000726..fab727d83a9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6118,10 +6118,10 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.1.1.tgz#c4b489e80096d9df1dfc97c79871aea7c617c469" integrity sha512-zAoAQiudy+r5SvnSw3KJy5os/oRJYHzrzja/tBDqrZtNhUw8bt6y8OBzMWcjWr+8liV8Eb6yOhw8WZ7VFZ5ZzA== -flow-bin@0.171.0: - version "0.171.0" - resolved "https://registry.yarnpkg.com/flow-bin/-/flow-bin-0.171.0.tgz#43902cf3ab10704a9c8a96bd16f789d92490ba1c" - integrity sha512-2HEiXAyE60ztGs+loFk6XSskL69THL6tSjzopUcbwgfrdbuZ5Jhv23qh1jUKP5AZJh0NNwxaFZ6To2p6xR+GEA== +flow-bin@0.173.0: + version "0.173.0" + resolved "https://packages.atlassian.com/api/npm/npm-remote/flow-bin/-/flow-bin-0.173.0.tgz#4eb4b0143ffcef3ef3ee64638554a8dff6b89735" + integrity sha512-CIvShSnB4I7SsfkE9S7ECpUCkBNqDQGDWr+0uzulKGYEBnCYwFzITE1T84weLmINQwO1dR6ntsH0LlWLoGCquQ== flush-write-stream@^1.0.0, flush-write-stream@^1.0.2: version "1.1.1" From ae3ee145388a40b0752771f5a8a8abc10a172c5e Mon Sep 17 00:00:00 2001 From: thebriando Date: Tue, 8 Mar 2022 20:17:08 -0800 Subject: [PATCH 27/87] Remove reachableBundles --- .../experimental/src/ExperimentalBundler.js | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index b4d370affa6..a0b7b2cbc67 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -265,11 +265,6 @@ function createIdealGraph( Array<[Dependency, Bundle]>, > = new DefaultMap(() => []); - // Connects bundleRoot assets to the assets that must be in the bundle - let reachableBundles: DefaultMap> = new DefaultMap( - () => new Set(), - ); - let bundleGraph: Graph = new Graph(); let stack: Array<[BundleRoot, NodeId]> = []; @@ -403,20 +398,6 @@ function createIdealGraph( ), dependencyPriorityEdges[dependency.priority], ); - - // Walk up the stack until we hit a different asset type - // and mark each bundle as reachable from every parent bundle - for (let i = stack.length - 1; i >= 0; i--) { - let [stackAsset] = stack[i]; - if ( - stackAsset.type !== childAsset.type || - stackAsset.env.context !== childAsset.env.context || - stackAsset.env.isIsolated() - ) { - break; - } - reachableBundles.get(stackAsset).add(childAsset); - } continue; } if ( From 7a98f044cbe57dc96ea5fbebcb2a4726c14be40b Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Wed, 9 Mar 2022 05:23:03 +0100 Subject: [PATCH 28/87] Bump lmdb (#7797) --- packages/core/cache/package.json | 2 +- yarn.lock | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/core/cache/package.json b/packages/core/cache/package.json index 9d90b365790..70181e28d46 100644 --- a/packages/core/cache/package.json +++ b/packages/core/cache/package.json @@ -27,7 +27,7 @@ "@parcel/fs": "2.3.2", "@parcel/logger": "2.3.2", "@parcel/utils": "2.3.2", - "lmdb": "2.2.3" + "lmdb": "2.2.4" }, "peerDependencies": { "@parcel/core": "^2.3.2" diff --git a/yarn.lock b/yarn.lock index fab727d83a9..3d6a543153d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8254,10 +8254,10 @@ listr2@^2.1.0: rxjs "^6.5.5" through "^2.3.8" -lmdb@2.2.3: - version "2.2.3" - resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.2.3.tgz#713ffa515c31e042808abf364b4aa0feaeaf6360" - integrity sha512-+OiHQpw22mBBxocb/9vcVNETqf0k5vgHA2r+KX7eCf8j5tSV50ZIv388iY1mnnrERIUhs2sjKQbZhPg7z4HyPQ== +lmdb@2.2.4: + version "2.2.4" + resolved "https://registry.yarnpkg.com/lmdb/-/lmdb-2.2.4.tgz#6494d5a1d1db152e0be759edcfa06893e4cbdb53" + integrity sha512-gto+BB2uEob8qRiTlOq+R3uX0YNHsX9mjxj9Sbdue/LIKqu6IlZjrsjKeGyOMquc/474GEqFyX2pdytpydp0rQ== dependencies: msgpackr "^1.5.4" nan "^2.14.2" From 968fde6d9636fb672cd4c892a15694d2f8cddad6 Mon Sep 17 00:00:00 2001 From: Niklas Mischkulnig <4586894+mischnic@users.noreply.github.com> Date: Wed, 9 Mar 2022 05:23:24 +0100 Subject: [PATCH 29/87] Replace typeof before DCE (#7788) --- .../test/integration/falsy-dep/index.js | 18 +++++++ packages/transformers/js/core/src/hoist.rs | 25 --------- packages/transformers/js/core/src/lib.rs | 6 +++ .../js/core/src/typeof_replacer.rs | 52 +++++++++++++++++++ 4 files changed, 76 insertions(+), 25 deletions(-) create mode 100644 packages/transformers/js/core/src/typeof_replacer.rs diff --git a/packages/core/integration-tests/test/integration/falsy-dep/index.js b/packages/core/integration-tests/test/integration/falsy-dep/index.js index d42ce74b972..9043f103482 100644 --- a/packages/core/integration-tests/test/integration/falsy-dep/index.js +++ b/packages/core/integration-tests/test/integration/falsy-dep/index.js @@ -24,4 +24,22 @@ if (process.env.NODE_ENV !== 'test') { require('./true-alternate'); } +if (typeof require === "function") { + require('./true-consequent'); +} else { + require('./false-alternate'); +} + +if (typeof exports === "object") { + require('./true-consequent'); +} else { + require('./false-alternate'); +} + +if (typeof module === "object") { + require('./true-consequent'); +} else { + require('./false-alternate'); +} + module.exports = 2; diff --git a/packages/transformers/js/core/src/hoist.rs b/packages/transformers/js/core/src/hoist.rs index 3d3eb58f331..02e17bce0b1 100644 --- a/packages/transformers/js/core/src/hoist.rs +++ b/packages/transformers/js/core/src/hoist.rs @@ -693,31 +693,6 @@ impl<'a> Fold for Hoist<'a> { } } } - Expr::Unary(ref unary) => { - // typeof require -> "function" - // typeof module -> "object" - if unary.op == UnaryOp::TypeOf { - if let Expr::Ident(ident) = &*unary.arg { - if ident.sym == js_word!("require") && !self.collect.decls.contains(&id!(ident)) { - return Expr::Lit(Lit::Str(Str { - kind: StrKind::Synthesized, - has_escape: false, - span: unary.span, - value: js_word!("function"), - })); - } - - if ident.sym == js_word!("module") && !self.collect.decls.contains(&id!(ident)) { - return Expr::Lit(Lit::Str(Str { - kind: StrKind::Synthesized, - has_escape: false, - span: unary.span, - value: js_word!("object"), - })); - } - } - } - } _ => {} } diff --git a/packages/transformers/js/core/src/lib.rs b/packages/transformers/js/core/src/lib.rs index 58ba9bd4cea..3e9c161f629 100644 --- a/packages/transformers/js/core/src/lib.rs +++ b/packages/transformers/js/core/src/lib.rs @@ -18,6 +18,7 @@ mod fs; mod global_replacer; mod hoist; mod modules; +mod typeof_replacer; mod utils; use std::collections::{HashMap, HashSet}; @@ -49,6 +50,7 @@ use fs::inline_fs; use global_replacer::GlobalReplacer; use hoist::{hoist, CollectResult, HoistResult}; use modules::esm2cjs; +use typeof_replacer::*; use utils::{CodeHighlight, Diagnostic, DiagnosticSeverity, SourceLocation, SourceType}; use crate::hoist::Collect; @@ -321,6 +323,10 @@ pub fn transform(config: Config) -> Result { let mut diagnostics = vec![]; let module = { let mut passes = chain!( + Optional::new( + TypeofReplacer { decls: &decls }, + config.source_type != SourceType::Script + ), // Inline process.env and process.browser Optional::new( EnvReplacer { diff --git a/packages/transformers/js/core/src/typeof_replacer.rs b/packages/transformers/js/core/src/typeof_replacer.rs new file mode 100644 index 00000000000..f76c1fa8bf7 --- /dev/null +++ b/packages/transformers/js/core/src/typeof_replacer.rs @@ -0,0 +1,52 @@ +use std::collections::HashSet; + +use swc_atoms::JsWord; +use swc_ecmascript::ast::{Expr, Lit, Str, StrKind, UnaryOp}; +use swc_ecmascript::visit::{Fold, FoldWith}; + +use crate::id; +use crate::utils::IdentId; + +pub struct TypeofReplacer<'a> { + pub decls: &'a HashSet, +} + +impl<'a> Fold for TypeofReplacer<'a> { + fn fold_expr(&mut self, node: Expr) -> Expr { + if let Expr::Unary(ref unary) = node { + // typeof require -> "function" + // typeof module -> "object" + if unary.op == UnaryOp::TypeOf { + if let Expr::Ident(ident) = &*unary.arg { + if ident.sym == js_word!("require") && !self.decls.contains(&id!(ident)) { + return Expr::Lit(Lit::Str(Str { + kind: StrKind::Synthesized, + has_escape: false, + span: unary.span, + value: js_word!("function"), + })); + } + let exports: JsWord = "exports".into(); + if ident.sym == exports && !self.decls.contains(&id!(ident)) { + return Expr::Lit(Lit::Str(Str { + kind: StrKind::Synthesized, + has_escape: false, + span: unary.span, + value: js_word!("object"), + })); + } + + if ident.sym == js_word!("module") && !self.decls.contains(&id!(ident)) { + return Expr::Lit(Lit::Str(Str { + kind: StrKind::Synthesized, + has_escape: false, + span: unary.span, + value: js_word!("object"), + })); + } + } + } + } + node.fold_children_with(self) + } +} From 813650091abbfa4736ea54403c2820f9043cd883 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Thu, 3 Mar 2022 13:07:24 -0800 Subject: [PATCH 30/87] Consider sibling availability before removing from ancestorAssets --- .../bundlers/experimental/src/ExperimentalBundler.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index a0b7b2cbc67..d9cfb1e87db 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -613,7 +613,7 @@ function createIdealGraph( BundleRoot, Map, > = new DefaultMap(() => new Map()); - let borrowedAssets: DefaultMap> = new DefaultMap( + let lentAssets: DefaultMap> = new DefaultMap( () => new Set(), ); @@ -629,6 +629,10 @@ function createIdealGraph( ancestorAssets.set(entry, new Set()); } + // Visit nodes in a topological order, visiting parent nodes before child nodes. + // This allows us to construct an understanding of which assets will already be + // loaded and available when a bundle runs, by pushing available assets downwards and + // computing the intersection of assets available through all possible paths to a bundle. for (let nodeId of asyncBundleRootGraph.topoSort()) { const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; @@ -694,14 +698,14 @@ function createIdealGraph( for (let asset of childAvailableAssets) { if (!available.has(asset)) { if (childrenAssets.has(asset)) { - borrowedAssets.get(childrenAssets.get(asset)[0]).add(asset); + lentAssets.get(childrenAssets.get(asset)[0]).add(asset); } else { childAvailableAssets.delete(asset); } } } } else { - ancestorAssets.set(child, new Set(available)); + asyncAncestorAssets.set(child, new Set(available)); } } } From a5d961001a8672b358832aa829141d2248b9fdb7 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Thu, 3 Mar 2022 15:02:25 -0800 Subject: [PATCH 31/87] Consider assets in siblings before duplicating --- .../experimental/src/ExperimentalBundler.js | 213 +++++++++--------- 1 file changed, 110 insertions(+), 103 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index d9cfb1e87db..c12762f25cc 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -559,6 +559,7 @@ function createIdealGraph( ((root.isBundleSplittable && !entries.has(root)) || isAsync || resolved.bundleBehavior === 'isolated' || + resolved.bundleBehavior === 'inline' || root.type !== resolved.type) ) { let rootNodeId = nullthrows(bundles.get(root.id)); @@ -608,25 +609,15 @@ function createIdealGraph( // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets - let ancestorAssets: Map> = new Map(); - let siblingAssets: DefaultMap< - BundleRoot, - Map, - > = new DefaultMap(() => new Map()); + let asyncAncestorAssets: Map> = new Map(); let lentAssets: DefaultMap> = new DefaultMap( () => new Set(), ); - // Reference count of each asset available within a given bundleRoot's bundle group - let assetRefsInBundleGroup: DefaultMap< - BundleRoot, - DefaultMap, - > = new DefaultMap(() => new DefaultMap(() => 0)); - // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group for (let entry of entries.keys()) { // Initialize an empty set of ancestors available to entries - ancestorAssets.set(entry, new Set()); + asyncAncestorAssets.set(entry, new Set()); } // Visit nodes in a topological order, visiting parent nodes before child nodes. @@ -637,35 +628,35 @@ function createIdealGraph( const bundleRoot = asyncBundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); - // First consider bundle group asset availability, processing only - // non-isolated bundles within that bundle group let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; - let available = new Set(ancestorAssets.get(bundleRoot)); - for (let bundleIdInGroup of [ - bundleGroupId, - ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), - ]) { - let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); - invariant(bundleInGroup !== 'root'); - if ( - bundleInGroup.bundleBehavior === 'isolated' || - bundleInGroup.bundleBehavior === 'inline' - ) { - continue; - } - let [siblingBundleRoot] = [...bundleInGroup.assets]; - // Assets directly connected to current bundleRoot - let assetsFromBundleRoot = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(siblingBundleRoot.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); + let available; + if (bundleRoot.bundleBehavior === 'isolated') { + available = new Set(); + } else { + available = new Set(asyncAncestorAssets.get(bundleRoot)); + for (let bundleIdInGroup of [ + bundleGroupId, + ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), + ]) { + let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); + invariant(bundleInGroup !== 'root'); + if ( + bundleInGroup.bundleBehavior === 'isolated' || + bundleInGroup.bundleBehavior === 'inline' + ) { + continue; + } + let [siblingBundleRoot] = [...bundleInGroup.assets]; + // Assets directly connected to current bundleRoot + let assetsFromBundleRoot = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(siblingBundleRoot.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); - for (let asset of [siblingBundleRoot, ...assetsFromBundleRoot]) { - available.add(asset); - if (bundleIdInGroup !== bundleGroupId) { - siblingAssets.get(bundleRoot).set(asset, siblingBundleRoot); + for (let asset of [siblingBundleRoot, ...assetsFromBundleRoot]) { + available.add(asset); } } } @@ -680,6 +671,12 @@ function createIdealGraph( for (let childId of children) { let child = asyncBundleRootGraph.getNode(childId); invariant(child !== 'root' && child != null); + if ( + child.bundleBehavior === 'isolated' || + child.bundleBehavior === 'inline' + ) { + continue; + } let assets = reachableRoots .getNodeIdsConnectedFrom(reachableRoots.getNodeIdByContentKey(child.id)) @@ -692,13 +689,23 @@ function createIdealGraph( for (let childId of children) { let child = asyncBundleRootGraph.getNode(childId); invariant(child !== 'root' && child != null); - const childAvailableAssets = ancestorAssets.get(child); + if ( + child.bundleBehavior === 'isolated' || + child.bundleBehavior === 'inline' + ) { + continue; + } + const childAvailableAssets = asyncAncestorAssets.get(child); if (childAvailableAssets != null) { for (let asset of childAvailableAssets) { if (!available.has(asset)) { - if (childrenAssets.has(asset)) { - lentAssets.get(childrenAssets.get(asset)[0]).add(asset); + let lender = childrenAssets + .get(asset) + ?.find(root => root !== child); + if (lender != null) { + // TODO: Unborrow if this is intersected away later + lentAssets.get(lender).add(asset); } else { childAvailableAssets.delete(asset); } @@ -726,32 +733,69 @@ function createIdealGraph( // meaning it may not be deduplicated. Otherwise, decrement all references in // the ancestry and keep it reachable = reachable.filter(b => { - let ancestry = ancestorAssets.get(b)?.get(asset); - if (ancestry === undefined) { - // No reachable bundles from this asset - return true; - } else if (ancestry === null) { - // Asset is reachable from this bundle + if (asyncAncestorAssets.get(b)?.has(asset)) { + // Asset is available asynchronously return false; - } else { - // If every bundle in its ancestry has more than 1 reference to the asset - if ( - ancestry.every( - bundleId => assetRefsInBundleGroup.get(bundleId).get(asset) > 1, - ) - ) { - for (let bundleRoot of ancestry) { - assetRefsInBundleGroup - .get(bundleRoot) - .set( - asset, - assetRefsInBundleGroup.get(bundleRoot).get(asset) - 1, - ); - } - return false; - } + } else if (lentAssets.get(b).has(asset)) { + // Some other bundle depends on this asset from us, and it's not already + // available in our async ancestry. We can't borrow it. + // TODO: Allow borrowing as long as it doesn't form a cycle? return true; } + + let canBorrow = false; + let potentialLenders = new Set(); + let bundleGroupIds = bundleGraph + .getNodeIdsConnectedTo(nullthrows(bundles.get(b.id))) + .filter(n => bundleGraph.getNode(n) !== 'root'); + if ( + bundleGraph.hasEdge( + nullthrows(bundleGraph.rootNodeId), + nullthrows(bundles.get(b.id)), + ) + ) { + bundleGroupIds.push(nullthrows(bundles.get(b.id))); + } + for (let bundleGroupId of bundleGroupIds) { + if (bundleGroupId === bundleGraph.rootNodeId) { + break; + } + + let lender = bundleGraph + .getNodeIdsConnectedFrom(bundleGroupId) + .map(id => bundleGraph.getNode(id)) + .find(siblingBundle => { + invariant(siblingBundle !== 'root' && siblingBundle != null); + return ( + [...siblingBundle.assets][0] !== b && + siblingBundle.bundleBehavior !== 'isolated' && + siblingBundle.bundleBehavior !== 'inline' && + reachableRoots.hasEdge( + reachableRoots.getNodeIdByContentKey( + [...siblingBundle.assets][0].id, + ), + reachableRoots.getNodeIdByContentKey(asset.id), + ) + ); + }); + if (lender == null) { + break; + } else { + invariant(typeof lender !== 'string'); + potentialLenders.add(lender); + } + canBorrow = true; + } + + if (canBorrow) { + for (let lender of potentialLenders) { + lentAssets.get([...lender.assets][0]).add(asset); + } + // Borrow this asset from siblings across each bundle group + return false; + } + + return true; }); let rootBundleTuple = bundleRoots.get(asset); @@ -790,7 +834,7 @@ function createIdealGraph( reachableRoots.hasEdge( reachableRoots.getNodeIdByContentKey(bundleRoot.id), reachableAssetId, - ) || ancestorAssets.get(bundleRoot)?.has(asset), + ) || asyncAncestorAssets.get(bundleRoot)?.has(asset), ); for (let bundleRoot of willInternalizeRoots) { @@ -821,7 +865,7 @@ function createIdealGraph( if (asyncBundleRootGraph.hasNode(asyncAssetId)) { asyncBundleRootGraph.removeNode(asyncAssetId); } - ancestorAssets.delete(asset); + asyncAncestorAssets.delete(asset); reachableRoots.replaceNodeIdsConnectedTo(reachableAssetId, []); } } @@ -1022,43 +1066,6 @@ async function loadBundlerConfig( }; } -function ancestryUnion( - ancestors: Set, - assetRefs: Map, - bundleRoot: BundleRoot, -): Map | null> { - let map = new Map(); - for (let a of ancestors) { - map.set(a, null); - } - for (let [asset, refCount] of assetRefs) { - if (!ancestors.has(asset) && refCount > 1) { - map.set(asset, [bundleRoot]); - } - } - return map; -} - -function ancestryIntersect( - currentMap: Map | null>, - map: Map | null>, -): void { - for (let [bundleRoot, currentAssets] of currentMap) { - if (map.has(bundleRoot)) { - let assets = map.get(bundleRoot); - if (assets) { - if (currentAssets) { - currentAssets.push(...assets); - } else { - currentMap.set(bundleRoot, [...assets]); - } - } - } else { - currentMap.delete(bundleRoot); - } - } -} - function getReachableBundleRoots(asset, graph): Array { return graph .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) From a7946d96950e2f8325132afcc15ac5bdec40cea2 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Thu, 10 Mar 2022 12:52:13 -0800 Subject: [PATCH 32/87] Remove unrelated change --- packages/packagers/html/src/HTMLPackager.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/packagers/html/src/HTMLPackager.js b/packages/packagers/html/src/HTMLPackager.js index dfc7057bcb5..b30faf3a110 100644 --- a/packages/packagers/html/src/HTMLPackager.js +++ b/packages/packagers/html/src/HTMLPackager.js @@ -57,7 +57,6 @@ export default (new Packager({ new Set(bundleGraph.getReferencedBundles(bundle, {recursive: false})), ), ]; - let renderConfig = config?.render; let {html} = await posthtml([ From eab212acf4ebf22fc2b5a2cf50ba4324325cd003 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Mon, 14 Mar 2022 13:19:49 -0700 Subject: [PATCH 33/87] Don't consider any of parent's async bundles as sibling --- .../experimental/src/ExperimentalBundler.js | 16 ++-------------- 1 file changed, 2 insertions(+), 14 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index c12762f25cc..68b8470e03e 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -19,7 +19,7 @@ import {ContentGraph, Graph} from '@parcel/graph'; import invariant from 'assert'; import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; -import {validateSchema, DefaultMap} from '@parcel/utils'; +import {setIntersect, validateSchema, DefaultMap} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -698,19 +698,7 @@ function createIdealGraph( const childAvailableAssets = asyncAncestorAssets.get(child); if (childAvailableAssets != null) { - for (let asset of childAvailableAssets) { - if (!available.has(asset)) { - let lender = childrenAssets - .get(asset) - ?.find(root => root !== child); - if (lender != null) { - // TODO: Unborrow if this is intersected away later - lentAssets.get(lender).add(asset); - } else { - childAvailableAssets.delete(asset); - } - } - } + setIntersect(childAvailableAssets, available); } else { asyncAncestorAssets.set(child, new Set(available)); } From 0cbd1f3c886653e47f8feab0906cb796ddd0c5dc Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 16 Mar 2022 16:28:27 -0700 Subject: [PATCH 34/87] Remove unused structure --- .../experimental/src/ExperimentalBundler.js | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 68b8470e03e..58bb7eee9da 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -661,31 +661,9 @@ function createIdealGraph( } } - let childrenAssets: DefaultMap> = new DefaultMap( - () => [], - ); let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); - // Group assets available across our children by the child. This will be used // to determine borrowers if needed below. - for (let childId of children) { - let child = asyncBundleRootGraph.getNode(childId); - invariant(child !== 'root' && child != null); - if ( - child.bundleBehavior === 'isolated' || - child.bundleBehavior === 'inline' - ) { - continue; - } - - let assets = reachableRoots - .getNodeIdsConnectedFrom(reachableRoots.getNodeIdByContentKey(child.id)) - .map(id => nullthrows(reachableRoots.getNode(id))); - for (let asset of [child, ...assets]) { - childrenAssets.get(asset).push(child); - } - } - for (let childId of children) { let child = asyncBundleRootGraph.getNode(childId); invariant(child !== 'root' && child != null); From 81ffca5e8ed0c70669003c3fd50582f448a2569a Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 22 Mar 2022 16:17:18 -0400 Subject: [PATCH 35/87] remove eager bundle reuse and related lending code --- .../experimental/src/ExperimentalBundler.js | 185 +----------------- 1 file changed, 6 insertions(+), 179 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 58bb7eee9da..c0480d8e91a 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -555,50 +555,17 @@ function createIdealGraph( let resolved = assets[0]; let isAsync = dependency.priority !== 'sync'; if ( - bundleRoots.has(resolved) && - ((root.isBundleSplittable && !entries.has(root)) || - isAsync || - resolved.bundleBehavior === 'isolated' || - resolved.bundleBehavior === 'inline' || - root.type !== resolved.type) + isAsync || + resolved.bundleBehavior === 'isolated' || + resolved.bundleBehavior === 'inline' || + root.type !== resolved.type ) { - let rootNodeId = nullthrows(bundles.get(root.id)); - let resolvedNodeId = nullthrows(bundles.get(resolved.id)); - if (!isAsync && resolved.bundleBehavior !== 'isolated') { - if (!bundleGraph.hasEdge(rootNodeId, resolvedNodeId)) { - bundleGraph.addEdge(rootNodeId, resolvedNodeId); - } - - // Reflect this connection in the async bundle root graph by - // connecting the reused bundle to every case where the original - // root bundle is loaded. This only necessary in cases that - // bundles in a group are executed in serial (e.g. js referenced - // by html) - for (let inboundAsync of asyncBundleRootGraph.getNodeIdsConnectedTo( - nullthrows(asyncBundleRootGraph.getNodeIdByContentKey(root.id)), - )) { - let resolvedInAsyncRootGraph = nullthrows( - asyncBundleRootGraph.getNodeIdByContentKey(resolved.id), - ); - if ( - !asyncBundleRootGraph.hasEdge( - inboundAsync, - resolvedInAsyncRootGraph, - ) - ) { - asyncBundleRootGraph.addEdge( - inboundAsync, - resolvedInAsyncRootGraph, - ); - } - } - } actions.skipChildren(); } return; } - + //asset node type let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( node.value.id, node.value, @@ -695,147 +662,7 @@ function createIdealGraph( // Filter out bundles from this asset's reachable array if // bundle does not contain the asset in its ancestry - // or if any of the bundles in the ancestry have a refcount of <= 1 for that asset, - // meaning it may not be deduplicated. Otherwise, decrement all references in - // the ancestry and keep it - reachable = reachable.filter(b => { - if (asyncAncestorAssets.get(b)?.has(asset)) { - // Asset is available asynchronously - return false; - } else if (lentAssets.get(b).has(asset)) { - // Some other bundle depends on this asset from us, and it's not already - // available in our async ancestry. We can't borrow it. - // TODO: Allow borrowing as long as it doesn't form a cycle? - return true; - } - - let canBorrow = false; - let potentialLenders = new Set(); - let bundleGroupIds = bundleGraph - .getNodeIdsConnectedTo(nullthrows(bundles.get(b.id))) - .filter(n => bundleGraph.getNode(n) !== 'root'); - if ( - bundleGraph.hasEdge( - nullthrows(bundleGraph.rootNodeId), - nullthrows(bundles.get(b.id)), - ) - ) { - bundleGroupIds.push(nullthrows(bundles.get(b.id))); - } - for (let bundleGroupId of bundleGroupIds) { - if (bundleGroupId === bundleGraph.rootNodeId) { - break; - } - - let lender = bundleGraph - .getNodeIdsConnectedFrom(bundleGroupId) - .map(id => bundleGraph.getNode(id)) - .find(siblingBundle => { - invariant(siblingBundle !== 'root' && siblingBundle != null); - return ( - [...siblingBundle.assets][0] !== b && - siblingBundle.bundleBehavior !== 'isolated' && - siblingBundle.bundleBehavior !== 'inline' && - reachableRoots.hasEdge( - reachableRoots.getNodeIdByContentKey( - [...siblingBundle.assets][0].id, - ), - reachableRoots.getNodeIdByContentKey(asset.id), - ) - ); - }); - if (lender == null) { - break; - } else { - invariant(typeof lender !== 'string'); - potentialLenders.add(lender); - } - canBorrow = true; - } - - if (canBorrow) { - for (let lender of potentialLenders) { - lentAssets.get([...lender.assets][0]).add(asset); - } - // Borrow this asset from siblings across each bundle group - return false; - } - - return true; - }); - - let rootBundleTuple = bundleRoots.get(asset); - if (rootBundleTuple != null) { - let rootBundle = nullthrows(bundleGraph.getNode(rootBundleTuple[0])); - invariant(rootBundle !== 'root'); - - if (!rootBundle.env.isIsolated()) { - if (!bundles.has(asset.id)) { - bundles.set(asset.id, rootBundleTuple[0]); - } - - for (let reachableAsset of reachable) { - if (reachableAsset !== asset) { - bundleGraph.addEdge( - nullthrows(bundleRoots.get(reachableAsset))[1], - rootBundleTuple[0], - ); - } - } - - let asyncAssetId = asyncBundleRootGraph.getNodeIdByContentKey(asset.id); - let loadedBy = asyncBundleRootGraph - .getNodeIdsConnectedTo(asyncAssetId) - .map(id => nullthrows(asyncBundleRootGraph.getNode(id))) - .filter(bundleRoot => bundleRoot !== 'root') - .map(bundleRoot => { - // For Flow - invariant(bundleRoot !== 'root'); - return bundleRoot; - }); - - let reachableAssetId = reachableRoots.getNodeIdByContentKey(asset.id); - let willInternalizeRoots = loadedBy.filter( - bundleRoot => - reachableRoots.hasEdge( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - reachableAssetId, - ) || asyncAncestorAssets.get(bundleRoot)?.has(asset), - ); - - for (let bundleRoot of willInternalizeRoots) { - if (bundleRoot !== asset) { - let bundle = nullthrows( - bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), - ); - invariant(bundle !== 'root'); - bundle.internalizedAssetIds.push(asset.id); - asyncBundleRootGraph.removeEdge( - asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), - asyncAssetId, - ); - } - } - - if ( - !entries.has(asset) && - loadedBy.length > 0 && - loadedBy.length === willInternalizeRoots.length - ) { - // The contents of this async bundle are accessible already in every - // use without it. Remove the async bundle entirely. - let bundleId = nullthrows(bundles.get(asset.id)); - bundleGraph.removeNode(bundleId); - bundles.delete(asset.id); - bundleRoots.delete(asset); - if (asyncBundleRootGraph.hasNode(asyncAssetId)) { - asyncBundleRootGraph.removeNode(asyncAssetId); - } - asyncAncestorAssets.delete(asset); - reachableRoots.replaceNodeIdsConnectedTo(reachableAssetId, []); - } - } - } + reachable = reachable.filter(b => !asyncAncestorAssets.get(b)?.has(asset)); if (reachable.length > 0) { let reachableEntries = reachable.filter( From 1c30b0a2c7c686e323d40e68d9bc1b5ec5b3d80b Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 23 Mar 2022 18:58:11 -0700 Subject: [PATCH 36/87] implement parallel request limits --- .../experimental/src/ExperimentalBundler.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 99719dba6c6..1a17e0e8672 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -789,6 +789,24 @@ function createIdealGraph( let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); let bundle; + + sourceBundles.filter(sourceBundleId => { + if (bundleId !== sourceBundleId) { + let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); + invariant(sourceBundle !== 'root'); + let bundleGroupIds = bundleGraph + .getNodeIdsConnectedTo(sourceBundleId) + .filter(n => bundleGraph.getNode(n) !== 'root'); + // Check that all bundle groups the source bundle belongs to + // are within the parallel request limit + return bundleGroupIds.every( + groupId => + bundleGraph.getNodeIdsConnectedFrom(groupId).length < + config.maxParallelRequests, + ); + } + }); + if (bundleId == null) { let firstSourceBundle = nullthrows( bundleGraph.getNode(sourceBundles[0]), @@ -808,7 +826,6 @@ function createIdealGraph( } bundle.assets.add(asset); bundle.size += asset.stats.size; - for (let sourceBundleId of sourceBundles) { if (bundleId !== sourceBundleId) { bundleGraph.addEdge(sourceBundleId, bundleId); From f8a8e65b9d2408c1ad8f443b550c13991e5da684 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Thu, 24 Mar 2022 16:42:38 -0700 Subject: [PATCH 37/87] cleanup --- packages/bundlers/experimental/src/ExperimentalBundler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 1a17e0e8672..5c7279c5a4a 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -788,7 +788,6 @@ function createIdealGraph( let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); - let bundle; sourceBundles.filter(sourceBundleId => { if (bundleId !== sourceBundleId) { @@ -807,6 +806,7 @@ function createIdealGraph( } }); + let bundle; if (bundleId == null) { let firstSourceBundle = nullthrows( bundleGraph.getNode(sourceBundles[0]), @@ -826,6 +826,7 @@ function createIdealGraph( } bundle.assets.add(asset); bundle.size += asset.stats.size; + for (let sourceBundleId of sourceBundles) { if (bundleId !== sourceBundleId) { bundleGraph.addEdge(sourceBundleId, bundleId); From 892edd37107a8032ae33466b721570efec3e556f Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Mon, 28 Mar 2022 18:09:24 -0700 Subject: [PATCH 38/87] create all shared bundles first then remove later --- .../experimental/src/ExperimentalBundler.js | 75 ++++++++++++++----- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 5c7279c5a4a..421722a1a73 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -788,24 +788,6 @@ function createIdealGraph( let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); - - sourceBundles.filter(sourceBundleId => { - if (bundleId !== sourceBundleId) { - let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); - invariant(sourceBundle !== 'root'); - let bundleGroupIds = bundleGraph - .getNodeIdsConnectedTo(sourceBundleId) - .filter(n => bundleGraph.getNode(n) !== 'root'); - // Check that all bundle groups the source bundle belongs to - // are within the parallel request limit - return bundleGroupIds.every( - groupId => - bundleGraph.getNodeIdsConnectedFrom(groupId).length < - config.maxParallelRequests, - ); - } - }); - let bundle; if (bundleId == null) { let firstSourceBundle = nullthrows( @@ -852,6 +834,63 @@ function createIdealGraph( } } + // Step 8: Remove shared bundles from bundle groups that hit the parallel request limit. + for (let [bundleId, bundleGroupId] of bundleRoots.values()) { + // Only handle bundle group entries. + if (bundleId != bundleGroupId) { + continue; + } + + // Find the bundles in this bundle group. + let bundleIdsInGroup = bundleGraph.getNodeIdsConnectedFrom(bundleGroupId); + if (bundleIdsInGroup.length > config.maxParallelRequests) { + // Sort the bundles so the smallest ones are removed first. + let bundlesInGroup = bundleIdsInGroup + .map(id => nullthrows(bundleGraph.getNode(id))) + .map(bundle => { + // For Flow + invariant(bundle !== 'root'); + return bundle; + }) + .sort((a, b) => a.size - b.size); + + // Remove bundles until the bundle group is within the parallel request limit. + for ( + let i = 0; + i < bundlesInGroup.length - config.maxParallelRequests; + i++ + ) { + let bundleId = bundleIdsInGroup[i]; + let bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); + // Add all assets in the shared bundle into the source bundles that are within this bundle group. + let sourceBundles = bundle.sourceBundles + .filter(b => bundlesInGroup.includes(b)) + .map(id => nullthrows(bundleGraph.getNode(id))); + + for (let sourceBundle of sourceBundles) { + invariant(sourceBundle !== 'root'); + for (let asset of bundle.assets) { + sourceBundle.assets.add(asset); + sourceBundle.size += asset.stats.size; + } + } + + // Remove the edge from this bundle group to the shared bundle. + bundleGraph.removeEdge(bundleGroupId, bundleId); + // If there is now only a single bundle group that contains this bundle, + // merge it into the remaining source bundles. If it is orphaned entirely, remove it. + let incomingNodeCount = + bundleGraph.getNodeIdsConnectedTo(bundleId).length; + if (incomingNodeCount === 1) { + removeBundle(bundleGraph, bundleId); + } else if (incomingNodeCount == 0) { + bundleGraph.removeNode(bundleId); + } + } + } + } + return { bundleGraph, dependencyBundleGraph, From b971d75dc12d513a28278a9f4779fdb9b44ea507 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 29 Mar 2022 17:17:29 -0400 Subject: [PATCH 39/87] Alter tests with mode production and correct assets with logic for splittable bundles --- .../experimental/src/ExperimentalBundler.js | 17 ++-- .../core/integration-tests/test/javascript.js | 85 ++++++++++--------- 2 files changed, 55 insertions(+), 47 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index c0480d8e91a..1cddf2ad3e3 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -577,9 +577,6 @@ function createIdealGraph( // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets let asyncAncestorAssets: Map> = new Map(); - let lentAssets: DefaultMap> = new DefaultMap( - () => new Set(), - ); // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group for (let entry of entries.keys()) { @@ -666,10 +663,20 @@ function createIdealGraph( if (reachable.length > 0) { let reachableEntries = reachable.filter( - a => entries.has(a) || !a.isBundleSplittable, + a => + entries.has(a) || + !a.isBundleSplittable || + getBundleFromBundleRoot(a).needsStableName || + getBundleFromBundleRoot(a).bundleBehavior === 'inline' || + getBundleFromBundleRoot(a).bundleBehavior === 'isolated', ); reachable = reachable.filter( - a => !entries.has(a) && a.isBundleSplittable, + a => + !entries.has(a) && + a.isBundleSplittable && + !getBundleFromBundleRoot(a).needsStableName && + getBundleFromBundleRoot(a).bundleBehavior !== 'inline' && + getBundleFromBundleRoot(a).bundleBehavior !== 'isolated', ); // Add assets to non-splittable bundles. diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 1108ef262ea..7ae91d39e1f 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -1014,49 +1014,42 @@ describe('javascript', function () { let b = await bundle( path.join(__dirname, '/integration/workers-module/index.js'), { + mode: 'production', defaultTargetOptions: { + shouldOptimize: false, shouldScopeHoist: true, }, }, ); - - if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { - assertBundles(b, [ - { - assets: ['dedicated-worker.js'], - }, - { - name: 'index.js', - assets: ['index.js', 'bundle-url.js', 'get-worker-url.js'], - }, - { - assets: ['shared-worker.js'], - }, - { - assets: ['index.js'], - }, - ]); - } else { - assertBundles(b, [ - { - assets: ['dedicated-worker.js', 'index.js'], - }, - { - name: 'index.js', - assets: ['index.js', 'bundle-url.js', 'get-worker-url.js'], - }, - { - assets: ['shared-worker.js', 'index.js'], - }, - ]); - } + assertBundles(b, [ + { + assets: ['dedicated-worker.js'], + }, + { + name: 'index.js', + assets: [ + 'index.js', + 'bundle-url.js', + 'get-worker-url.js', + 'bundle-manifest.js', + ], + }, + { + assets: ['shared-worker.js'], + }, + { + assets: ['index.js'], + }, + ]); let dedicated, shared; b.traverseBundles((bundle, ctx, traversal) => { - if (bundle.getMainEntry()?.filePath.endsWith('shared-worker.js')) { + let mainEntry = bundle.getMainEntry(); + if (mainEntry && mainEntry.filePath.endsWith('shared-worker.js')) { shared = bundle; } else if ( - bundle.getMainEntry()?.filePath.endsWith('dedicated-worker.js') + mainEntry && + mainEntry.filePath.endsWith('dedicated-worker.js') ) { dedicated = bundle; } @@ -1082,7 +1075,9 @@ describe('javascript', function () { let b = await bundle( path.join(__dirname, '/integration/workers-module/index.js'), { + mode: 'production', defaultTargetOptions: { + shouldOptimize: false, shouldScopeHoist, engines: { browsers: '>= 0.25%', @@ -1093,31 +1088,36 @@ describe('javascript', function () { assertBundles(b, [ { - assets: [ - 'dedicated-worker.js', - !shouldScopeHoist && 'esmodule-helpers.js', - 'index.js', - ].filter(Boolean), + assets: ['dedicated-worker.js'], }, { name: 'index.js', - assets: ['index.js', 'bundle-url.js', 'get-worker-url.js'], + assets: [ + 'index.js', + 'bundle-url.js', + 'get-worker-url.js', + 'bundle-manifest.js', + ], }, { assets: [ !shouldScopeHoist && 'esmodule-helpers.js', - 'shared-worker.js', 'index.js', ].filter(Boolean), }, + { + assets: ['shared-worker.js'], + }, ]); let dedicated, shared; b.traverseBundles((bundle, ctx, traversal) => { - if (bundle.getMainEntry().filePath.endsWith('shared-worker.js')) { + let mainEntry = bundle.getMainEntry(); + if (mainEntry && mainEntry.filePath.endsWith('shared-worker.js')) { shared = bundle; } else if ( - bundle.getMainEntry().filePath.endsWith('dedicated-worker.js') + mainEntry && + mainEntry.filePath.endsWith('dedicated-worker.js') ) { dedicated = bundle; } @@ -1973,6 +1973,7 @@ describe('javascript', function () { }); it('should contain duplicate assets in workers when in development', async () => { + if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) return; let b = await bundle( path.join(__dirname, '/integration/worker-shared/index.js'), {mode: 'development'}, From 01dd5119c4e615f38e2e17f07248dd5b7317c217 Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Tue, 29 Mar 2022 17:00:28 -0400 Subject: [PATCH 40/87] Skip unused dependencies in experimental bundler --- packages/bundlers/experimental/src/ExperimentalBundler.js | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 1cddf2ad3e3..de71f3306be 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -328,7 +328,7 @@ function createIdealGraph( // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, // only adding the entry asset of each bundle, not the subgraph. assetGraph.traverse({ - enter(node, context) { + enter(node, context, actions) { if (node.type === 'asset') { assets.push(node.value); @@ -343,6 +343,11 @@ function createIdealGraph( } let dependency = node.value; + if (assetGraph.isDependencySkipped(dependency)) { + actions.skipChildren(); + return node; + } + invariant(context?.type === 'asset'); let parentAsset = context.value; From d9049a17010ca50136d998e80d0d976c6e559b39 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 29 Mar 2022 14:28:55 -0700 Subject: [PATCH 41/87] Implement getBundleFromBundleRoot --- packages/bundlers/experimental/src/ExperimentalBundler.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index de71f3306be..bb4bfa9dd92 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -744,6 +744,14 @@ function createIdealGraph( } } + function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { + let bundle = bundleGraph.getNode( + nullthrows(bundleRoots.get(bundleRoot))[0], + ); + invariant(bundle !== 'root' && bundle != null); + return bundle; + } + return { bundleGraph, dependencyBundleGraph, From 76214cb286f4a5c913d4f50705423a8d9c677abf Mon Sep 17 00:00:00 2001 From: Devon Govett Date: Sat, 26 Mar 2022 12:02:04 -0400 Subject: [PATCH 42/87] Only add dependencies to CSS module JS, not CSS --- .../bundlers/default/src/DefaultBundler.js | 30 +++- .../experimental/src/ExperimentalBundler.js | 76 +++++++--- .../integration-tests/test/css-modules.js | 139 ++++++++++++++++++ .../css-modules-import/a.module.css | 7 + .../css-modules-import/b.module.css | 7 + .../css-modules-import/index.module.css | 6 + .../integration/css-modules-import/page1.html | 1 + .../integration/css-modules-import/page1.js | 4 + .../integration/css-modules-import/page2.html | 1 + .../integration/css-modules-import/page2.js | 4 + .../integration/css-modules-import/yarn.lock | 0 .../css-modules-targets/foo.module.css | 3 + .../css-modules-targets/index.html | 1 + .../integration/css-modules-targets/index.js | 5 + .../css-modules-targets/package.json | 3 + .../integration/css-modules-targets/yarn.lock | 0 .../transformers/css/src/CSSTransformer.js | 49 ++---- 17 files changed, 277 insertions(+), 59 deletions(-) create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/a.module.css create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/b.module.css create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/index.module.css create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/page1.html create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/page1.js create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/page2.html create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/page2.js create mode 100644 packages/core/integration-tests/test/integration/css-modules-import/yarn.lock create mode 100644 packages/core/integration-tests/test/integration/css-modules-targets/foo.module.css create mode 100644 packages/core/integration-tests/test/integration/css-modules-targets/index.html create mode 100644 packages/core/integration-tests/test/integration/css-modules-targets/index.js create mode 100644 packages/core/integration-tests/test/integration/css-modules-targets/package.json create mode 100644 packages/core/integration-tests/test/integration/css-modules-targets/yarn.lock diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index aa3a00320a6..6feb5c6fe6e 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -7,12 +7,13 @@ import type { Config, MutableBundleGraph, PluginOptions, + Dependency, } from '@parcel/types'; import type {SchemaEntity} from '@parcel/utils'; import invariant from 'assert'; import {Bundler} from '@parcel/plugin'; -import {validateSchema} from '@parcel/utils'; +import {validateSchema, DefaultMap} from '@parcel/utils'; import {hashString} from '@parcel/hash'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -54,6 +55,10 @@ export default (new Bundler({ bundle({bundleGraph, config}) { let bundleRoots: Map> = new Map(); let bundlesByEntryAsset: Map = new Map(); + let assetsToAddOnExit: DefaultMap< + Dependency, + Array<[Bundle, Asset]>, + > = new DefaultMap(() => []); // Step 1: create bundles for each of the explicit code split points. bundleGraph.traverse({ @@ -157,7 +162,7 @@ export default (new Bundler({ if (existingBundle) { // If a bundle of this type has already been created in this group, // merge this subgraph into it. - nullthrows(bundleRoots.get(existingBundle)).push(asset); + assetsToAddOnExit.get(node.value).push([existingBundle, asset]); bundlesByEntryAsset.set(asset, existingBundle); bundleGraph.createAssetReference(dependency, asset, existingBundle); } else { @@ -181,10 +186,8 @@ export default (new Bundler({ bundlesByEntryAsset.set(asset, bundle); bundleGraph.createAssetReference(dependency, asset, bundle); - // The bundle may have already been created, and the graph gave us back the original one... - if (!bundleRoots.has(bundle)) { - bundleRoots.set(bundle, [asset]); - } + // Queue the asset to be added on exit of this node, so we add dependencies first. + assetsToAddOnExit.get(node.value).push([bundle, asset]); } } @@ -193,6 +196,21 @@ export default (new Bundler({ parentNode: node, }; }, + exit: node => { + if (node.type === 'dependency' && assetsToAddOnExit.has(node.value)) { + let assetsToAdd = assetsToAddOnExit.get(node.value); + for (let [bundle, asset] of assetsToAdd) { + let root = bundleRoots.get(bundle); + if (root) { + root.push(asset); + } else { + bundleRoots.set(bundle, [asset]); + } + } + + assetsToAddOnExit.delete(node.value); + } + }, }); for (let [bundle, rootAssets] of bundleRoots) { diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index bb4bfa9dd92..e86da78f12b 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -53,6 +53,7 @@ const HTTP_OPTIONS = { type AssetId = string; type BundleRoot = Asset; export type Bundle = {| + uniqueKey: ?string, assets: Set, internalizedAssetIds: Array, bundleBehavior?: ?BundleBehavior, @@ -165,6 +166,17 @@ function decorateLegacyGraph( env: idealBundle.env, }), ); + } else if (idealBundle.uniqueKey != null) { + bundle = nullthrows( + bundleGraph.createBundle({ + uniqueKey: idealBundle.uniqueKey, + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + type: idealBundle.type, + target: idealBundle.target, + env: idealBundle.env, + }), + ); } else { bundle = nullthrows( bundleGraph.createBundle({ @@ -324,6 +336,10 @@ function createIdealGraph( } let assets = []; + let assetsToAddOnExit: DefaultMap< + Dependency, + Array<[Bundle, Asset]>, + > = new DefaultMap(() => []); // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, // only adding the entry asset of each bundle, not the subgraph. @@ -420,10 +436,16 @@ function createIdealGraph( // Find an existing bundle of the same type within the bundle group. let bundleId; + let entryAsset; + let uniqueKey; if ( childAsset.bundleBehavior !== 'inline' && dependency.priority !== 'parallel' ) { + uniqueKey = childAsset.id; + // TODO: share bundles even across different bundle groups by looking if the child + // asset is already a bundle root. In order for this to work, bundleRoots must be + // keyed by asset + target, not just asset, so that bundles are not shared between targets. bundleId = bundleGroup.type == childAsset.type ? bundleGroupNodeId @@ -433,6 +455,8 @@ function createIdealGraph( let node = bundleGraph.getNode(id); return node !== 'root' && node?.type == childAsset.type; }); + } else { + entryAsset = childAsset; } let bundle; @@ -445,7 +469,13 @@ function createIdealGraph( // Create a new bundle if none of the same type exists already. bundle = createBundle({ - asset: childAsset, + // We either have an entry asset or a unique key. + // Bundles created from type changes shouldn't have an entry asset. + asset: entryAsset, + uniqueKey, + type: childAsset.type, + env: childAsset.env, + bundleBehavior: childAsset.bundleBehavior, target: bundleGroup.target, needsStableName: childAsset.bundleBehavior === 'inline' || @@ -460,7 +490,11 @@ function createIdealGraph( // Otherwise, merge this asset into the existing bundle. bundle = bundleGraph.getNode(bundleId); invariant(bundle != null && bundle !== 'root'); - bundle.assets.add(childAsset); + } + + if (!entryAsset) { + // Queue the asset to be added on exit of this node, so we add dependencies first. + assetsToAddOnExit.get(dependency).push([bundle, childAsset]); } bundles.set(childAsset.id, bundleId); @@ -499,6 +533,15 @@ function createIdealGraph( return node; }, exit(node) { + if (node.type === 'dependency' && assetsToAddOnExit.has(node.value)) { + let assetsToAdd = assetsToAddOnExit.get(node.value); + for (let [bundle, asset] of assetsToAdd) { + bundle.assets.add(asset); + bundle.size += asset.stats.size; + } + assetsToAddOnExit.delete(node.value); + } + if (stack[stack.length - 1]?.[0] === node.value) { stack.pop(); } @@ -781,26 +824,18 @@ const CONFIG_SCHEMA: SchemaEntity = { additionalProperties: false, }; -function createBundle( - opts: - | {| - target: Target, - env: Environment, - type: string, - needsStableName?: boolean, - bundleBehavior?: ?BundleBehavior, - |} - | {| - target: Target, - asset: Asset, - env?: Environment, - type?: string, - needsStableName?: boolean, - bundleBehavior?: ?BundleBehavior, - |}, -): Bundle { +function createBundle(opts: {| + uniqueKey?: string, + target: Target, + asset?: Asset, + env?: Environment, + type?: string, + needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, +|}): Bundle { if (opts.asset == null) { return { + uniqueKey: opts.uniqueKey, assets: new Set(), internalizedAssetIds: [], size: 0, @@ -815,6 +850,7 @@ function createBundle( let asset = nullthrows(opts.asset); return { + uniqueKey: opts.uniqueKey, assets: new Set([asset]), internalizedAssetIds: [], size: asset.stats.size, diff --git a/packages/core/integration-tests/test/css-modules.js b/packages/core/integration-tests/test/css-modules.js index f591e517873..3ba4a0a0352 100644 --- a/packages/core/integration-tests/test/css-modules.js +++ b/packages/core/integration-tests/test/css-modules.js @@ -3,6 +3,7 @@ import path from 'path'; import { bundle, run, + runBundle, assertBundles, distDir, outputFS, @@ -448,4 +449,142 @@ describe('css modules', () => { let css = await outputFS.readFile(path.join(distDir, 'index.css'), 'utf8'); assert(css.includes('color: red')); }); + + it('should optimize away unused @keyframes', async function () { + let b = await bundle( + path.join(__dirname, '/integration/css-modules-keyframes/index.js'), + { + mode: 'production', + }, + ); + + assertBundles(b, [ + { + name: 'index.js', + assets: ['index.js', 'index.module.css'], + }, + { + name: 'index.css', + assets: ['index.module.css'], + }, + ]); + + let css = await outputFS.readFile(path.join(distDir, 'index.css'), 'utf8'); + assert(css.includes('@keyframes test')); + assert(!css.includes('@keyframes unused')); + }); + + it('should not double optimize css modules processed with postcss', async function () { + let b = await bundle( + path.join(__dirname, '/integration/postcss-modules-optimize/index.js'), + { + mode: 'production', + }, + ); + + assertBundles(b, [ + { + name: 'index.js', + assets: ['index.js', 'index.css'], + }, + { + name: 'index.css', + assets: ['index.css'], + }, + ]); + + let css = await outputFS.readFile(path.join(distDir, 'index.css'), 'utf8'); + assert(css.includes('@keyframes test')); + assert(css.includes('@keyframes unused')); + }); + + it('should compile css modules for multiple targets', async function () { + let b = await bundle( + path.join(__dirname, '/integration/css-modules-targets/index.html'), + { + mode: 'production', + }, + ); + + assertBundles(b, [ + { + name: 'index.html', + assets: ['index.html'], + }, + { + type: 'js', + assets: ['index.js', 'foo.module.css'], + }, + { + type: 'js', + assets: ['index.js', 'foo.module.css'], + }, + { + type: 'css', + assets: ['foo.module.css'], + }, + ]); + }); + + it('should handle @import in css modules', async function () { + let b = await bundle([ + path.join(__dirname, '/integration/css-modules-import/page1.html'), + path.join(__dirname, '/integration/css-modules-import/page2.html'), + ]); + + let res = []; + await runBundle( + b, + b.getBundles().find(b => b.name === 'page1.html'), + { + sideEffect: s => res.push(s), + }, + ); + + assert.deepEqual(res, ['page1']); + + res = []; + await runBundle( + b, + b.getBundles().find(b => b.name === 'page2.html'), + { + sideEffect: s => res.push(s), + }, + ); + + assert.deepEqual(res, ['page2']); + + assertBundles(b, [ + { + name: 'page1.html', + assets: ['page1.html'], + }, + { + name: 'page2.html', + assets: ['page2.html'], + }, + { + type: 'js', + assets: [ + 'page1.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'js', + assets: [ + 'page2.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'css', + assets: ['index.module.css', 'a.module.css', 'b.module.css'], + }, + ]); + }); }); diff --git a/packages/core/integration-tests/test/integration/css-modules-import/a.module.css b/packages/core/integration-tests/test/integration/css-modules-import/a.module.css new file mode 100644 index 00000000000..087615fb180 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-import/a.module.css @@ -0,0 +1,7 @@ +.a { + color: red; +} + +.foo { + background: green; +} diff --git a/packages/core/integration-tests/test/integration/css-modules-import/b.module.css b/packages/core/integration-tests/test/integration/css-modules-import/b.module.css new file mode 100644 index 00000000000..367397f5e48 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-import/b.module.css @@ -0,0 +1,7 @@ +.b { + color: purple; +} + +.foo { + font-family: Helvetica; +} diff --git a/packages/core/integration-tests/test/integration/css-modules-import/index.module.css b/packages/core/integration-tests/test/integration/css-modules-import/index.module.css new file mode 100644 index 00000000000..bc51dd98cba --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-import/index.module.css @@ -0,0 +1,6 @@ +@import "./a.module.css"; +@import "./b.module.css"; + +.foo { + color: red; +} diff --git a/packages/core/integration-tests/test/integration/css-modules-import/page1.html b/packages/core/integration-tests/test/integration/css-modules-import/page1.html new file mode 100644 index 00000000000..ca4432a469d --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-import/page1.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/core/integration-tests/test/integration/css-modules-import/page1.js b/packages/core/integration-tests/test/integration/css-modules-import/page1.js new file mode 100644 index 00000000000..fdef88b4d42 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-import/page1.js @@ -0,0 +1,4 @@ +import * as foo from './index.module.css'; + +sideEffect('page1'); +module.exports = foo; diff --git a/packages/core/integration-tests/test/integration/css-modules-import/page2.html b/packages/core/integration-tests/test/integration/css-modules-import/page2.html new file mode 100644 index 00000000000..e930f5be7b0 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-import/page2.html @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/packages/core/integration-tests/test/integration/css-modules-import/page2.js b/packages/core/integration-tests/test/integration/css-modules-import/page2.js new file mode 100644 index 00000000000..5e6315cb553 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-import/page2.js @@ -0,0 +1,4 @@ +import * as foo from './index.module.css'; + +sideEffect('page2'); +module.exports = foo; diff --git a/packages/core/integration-tests/test/integration/css-modules-import/yarn.lock b/packages/core/integration-tests/test/integration/css-modules-import/yarn.lock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/core/integration-tests/test/integration/css-modules-targets/foo.module.css b/packages/core/integration-tests/test/integration/css-modules-targets/foo.module.css new file mode 100644 index 00000000000..a15c877ac01 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-targets/foo.module.css @@ -0,0 +1,3 @@ +.foo { + color: red; +} diff --git a/packages/core/integration-tests/test/integration/css-modules-targets/index.html b/packages/core/integration-tests/test/integration/css-modules-targets/index.html new file mode 100644 index 00000000000..28a01ea786f --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-targets/index.html @@ -0,0 +1 @@ + diff --git a/packages/core/integration-tests/test/integration/css-modules-targets/index.js b/packages/core/integration-tests/test/integration/css-modules-targets/index.js new file mode 100644 index 00000000000..4eae1a3de99 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-targets/index.js @@ -0,0 +1,5 @@ +import * as foo from './foo.module.css'; + +module.exports = function () { + return foo.foo; +}; diff --git a/packages/core/integration-tests/test/integration/css-modules-targets/package.json b/packages/core/integration-tests/test/integration/css-modules-targets/package.json new file mode 100644 index 00000000000..6eb5e660335 --- /dev/null +++ b/packages/core/integration-tests/test/integration/css-modules-targets/package.json @@ -0,0 +1,3 @@ +{ + "browserslist": "IE >= 11" +} diff --git a/packages/core/integration-tests/test/integration/css-modules-targets/yarn.lock b/packages/core/integration-tests/test/integration/css-modules-targets/yarn.lock new file mode 100644 index 00000000000..e69de29bb2d diff --git a/packages/transformers/css/src/CSSTransformer.js b/packages/transformers/css/src/CSSTransformer.js index 0cf252b731b..b1a14d8bd4d 100644 --- a/packages/transformers/css/src/CSSTransformer.js +++ b/packages/transformers/css/src/CSSTransformer.js @@ -94,7 +94,7 @@ export default (new Transformer({ loc = remapSourceLocation(loc, originalMap); } - if (dep.type === 'import') { + if (dep.type === 'import' && !res.exports) { asset.addDependency({ specifier: dep.url, specifierType: 'url', @@ -104,7 +104,6 @@ export default (new Transformer({ isCSSImport: true, media: dep.media, }, - symbols: new Map([['*', {local: '*', isWeak: true, loc}]]), }); } else if (dep.type === 'url') { asset.addURLDependency(dep.url, { @@ -125,7 +124,6 @@ export default (new Transformer({ asset.symbols.set('default', 'default'); let dependencies = new Map(); - let selfReferences = new Set(); let locals = new Map(); let c = 0; let depjs = ''; @@ -147,10 +145,6 @@ export default (new Transformer({ let e = exports[key]; let s = `module.exports[${JSON.stringify(key)}] = \`${e.name}`; - if (e.isReferenced) { - selfReferences.add(e.name); - } - for (let ref of e.composes) { s += ' '; if (ref.type === 'local') { @@ -171,17 +165,19 @@ export default (new Transformer({ ref.specifier, )};\n`; dependencies.set(ref.specifier, d); - - asset.addDependency({ - specifier: ref.specifier, - specifierType: 'url', - }); } s += '${' + `${d}[${JSON.stringify(ref.name)}]` + '}'; } } s += '`;\n'; + + // If the export is referenced internally (e.g. used @keyframes), add a self-reference + // to the JS so the symbol is retained during tree-shaking. + if (e.isReferenced) { + s += `module.exports[${JSON.stringify(key)}];\n`; + } + js += s; }; @@ -190,12 +186,14 @@ export default (new Transformer({ add(key); } - for (let dep of asset.getDependencies()) { - if (dep.priority === 'sync') { - // TODO: Figure out how to treeshake this - let d = `dep_$${c++}`; - depjs += `import * as ${d} from ${JSON.stringify(dep.specifier)};\n`; - depjs += `for (let key in ${d}) { if (key in module.exports) module.exports[key] += ' ' + ${d}[key]; else module.exports[key] = ${d}[key]; }\n`; + if (res.dependencies) { + for (let dep of res.dependencies) { + if (dep.type === 'import') { + // TODO: Figure out how to treeshake this + let d = `dep_$${c++}`; + depjs += `import * as ${d} from ${JSON.stringify(dep.url)};\n`; + depjs += `for (let key in ${d}) { if (key in module.exports) module.exports[key] += ' ' + ${d}[key]; else module.exports[key] = ${d}[key]; }\n`; + } } } @@ -205,21 +203,6 @@ export default (new Transformer({ dependencies: jsDeps, env: asset.env, }); - - if (selfReferences.size > 0) { - asset.addDependency({ - specifier: `./${path.basename(asset.filePath)}`, - specifierType: 'url', - symbols: new Map( - [...locals] - .filter(([local]) => selfReferences.has(local)) - .map(([local, exported]) => [ - exported, - {local, isWeak: false, loc: null}, - ]), - ), - }); - } } // Normalize the asset's environment so that properties that only affect JS don't cause CSS to be duplicated. From 7d203429c01b54ad86477bf84193ae392b84dbaf Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Wed, 30 Mar 2022 15:07:37 -0700 Subject: [PATCH 43/87] Handle multiple assets on dependencies in reachability --- .../experimental/src/ExperimentalBundler.js | 89 +++++++++---------- 1 file changed, 42 insertions(+), 47 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e86da78f12b..e45c7c07777 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -565,60 +565,55 @@ function createIdealGraph( return; } - if (node.type === 'dependency') { - let dependency = node.value; - - if (dependencyBundleGraph.hasContentKey(dependency.id)) { - if (dependency.priority !== 'sync') { - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return; - } - - invariant(assets.length === 1); - let bundleRoot = assets[0]; - let bundle = nullthrows( - bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), - ); - if ( - bundle !== 'root' && - bundle.bundleBehavior !== 'isolated' && - bundle.bundleBehavior !== 'inline' && - !bundle.env.isIsolated() - ) { - asyncBundleRootGraph.addEdge( - asyncBundleRootGraph.getNodeIdByContentKey(root.id), - asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), - ); - } - } - } - - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return; - } - - invariant(assets.length === 1); - let resolved = assets[0]; - let isAsync = dependency.priority !== 'sync'; + if (node.type === 'asset') { + let asset = node.value; if ( - isAsync || - resolved.bundleBehavior === 'isolated' || - resolved.bundleBehavior === 'inline' || - root.type !== resolved.type + asset.bundleBehavior === 'isolated' || + asset.bundleBehavior === 'inline' || + root.type !== asset.type ) { actions.skipChildren(); + return; } + let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( + node.value.id, + node.value, + ); + reachableRoots.addEdge(rootNodeId, nodeId); return; } - //asset node type - let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( - node.value.id, - node.value, - ); - reachableRoots.addEdge(rootNodeId, nodeId); + + let dependency = node.value; + if (dependencyBundleGraph.hasContentKey(dependency.id)) { + if (dependency.priority !== 'sync') { + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return; + } + + invariant(assets.length === 1); + let bundleRoot = assets[0]; + let bundle = nullthrows( + bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), + ); + if ( + bundle !== 'root' && + bundle.bundleBehavior !== 'isolated' && + bundle.bundleBehavior !== 'inline' && + !bundle.env.isIsolated() + ) { + asyncBundleRootGraph.addEdge( + asyncBundleRootGraph.getNodeIdByContentKey(root.id), + asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + ); + } + } + } + + if (dependency.priority !== 'sync') { + actions.skipChildren(); + } }, root); } From 5107bd58a53b1babe82adf5ba8b74d3e8a737cf9 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Thu, 31 Mar 2022 11:59:09 -0700 Subject: [PATCH 44/87] ScopeHoistingPackager: Handle different wrapped ancestries in wrapping dfs --- packages/core/core/src/BundleGraph.js | 9 +++++++-- packages/core/core/src/public/Bundle.js | 6 +++++- packages/core/types/index.js | 5 ++++- .../packagers/js/src/ScopeHoistingPackager.js | 16 +++++++++++++--- 4 files changed, 29 insertions(+), 7 deletions(-) diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 537ecc0f78a..3f0de396c84 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -860,10 +860,12 @@ export default class BundleGraph { traverseAssets( bundle: Bundle, visit: GraphVisitor, + startAsset?: Asset, ): ?TContext { return this.traverseBundle( bundle, mapVisitor(node => (node.type === 'asset' ? node.value : null), visit), + startAsset, ); } @@ -1057,8 +1059,9 @@ export default class BundleGraph { traverseBundle( bundle: Bundle, visit: GraphVisitor, + startAsset?: Asset, ): ?TContext { - let entries = true; + let entries = !startAsset; let bundleNodeId = this._graph.getNodeIdByContentKey(bundle.id); // A modified DFS traversal which traverses entry assets in the same order @@ -1085,7 +1088,9 @@ export default class BundleGraph { actions.skipChildren(); }, visit), - startNodeId: bundleNodeId, + startNodeId: startAsset + ? this._graph.getNodeIdByContentKey(startAsset.id) + : bundleNodeId, getChildren: nodeId => { let children = this._graph .getNodeIdsConnectedFrom(nodeId) diff --git a/packages/core/core/src/public/Bundle.js b/packages/core/core/src/public/Bundle.js index 3581678d33d..fb95d1a5359 100644 --- a/packages/core/core/src/public/Bundle.js +++ b/packages/core/core/src/public/Bundle.js @@ -186,10 +186,14 @@ export class Bundle implements IBundle { ); } - traverseAssets(visit: GraphVisitor): ?TContext { + traverseAssets( + visit: GraphVisitor, + startAsset?: IAsset, + ): ?TContext { return this.#bundleGraph.traverseAssets( this.#bundle, mapVisitor(asset => assetFromValue(asset, this.#options), visit), + startAsset ? assetToAssetValue(startAsset) : undefined, ); } } diff --git a/packages/core/types/index.js b/packages/core/types/index.js index c79df875b66..5cc58c64d1c 100644 --- a/packages/core/types/index.js +++ b/packages/core/types/index.js @@ -1243,7 +1243,10 @@ export interface Bundle { /** Returns whether the bundle includes the given dependency. */ hasDependency(Dependency): boolean; /** Traverses the assets in the bundle. */ - traverseAssets(visit: GraphVisitor): ?TContext; + traverseAssets( + visit: GraphVisitor, + startAsset?: Asset, + ): ?TContext; /** Traverses assets and dependencies in the bundle. */ traverse( visit: GraphVisitor, diff --git a/packages/packagers/js/src/ScopeHoistingPackager.js b/packages/packagers/js/src/ScopeHoistingPackager.js index 87ed3952bd1..36cc5422e5a 100644 --- a/packages/packagers/js/src/ScopeHoistingPackager.js +++ b/packages/packagers/js/src/ScopeHoistingPackager.js @@ -241,7 +241,7 @@ export class ScopeHoistingPackager { async loadAssets(): Promise> { let queue = new PromiseQueue({maxConcurrent: 32}); let wrapped = []; - this.bundle.traverseAssets((asset, shouldWrap) => { + this.bundle.traverseAssets(asset => { queue.add(async () => { let [code, map] = await Promise.all([ asset.getCode(), @@ -251,7 +251,6 @@ export class ScopeHoistingPackager { }); if ( - shouldWrap || asset.meta.shouldWrap || this.isAsyncBundle || this.bundle.env.sourceType === 'script' || @@ -262,10 +261,21 @@ export class ScopeHoistingPackager { ) { this.wrappedAssets.add(asset.id); wrapped.push(asset); - return true; } }); + for (let wrappedAsset of [...wrapped]) { + this.bundle.traverseAssets((asset, _, actions) => { + if (asset !== wrappedAsset && this.wrappedAssets.has(asset.id)) { + actions.skipChildren(); + return; + } + + this.wrappedAssets.add(asset.id); + wrapped.push(asset); + }, wrappedAsset); + } + this.assetOutputs = new Map(await queue.run()); return wrapped; } From 403ae53b20ca7175e98802dbc7f369fb49aa0b2d Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 5 Apr 2022 12:31:10 -0400 Subject: [PATCH 45/87] move reachable root creation earlier to prevent unnecessary async bundle --- .../experimental/src/ExperimentalBundler.js | 101 ++++++++++++++++-- 1 file changed, 95 insertions(+), 6 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e86da78f12b..abd1418f38f 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -289,6 +289,10 @@ function createIdealGraph( let entries: Map = new Map(); let sharedToSourceBundleIds: Map> = new Map(); + // Attempt to replace reachable roots by building it earlier. + // this models bundleRoots and the assets that require it synchronously + let syncRootsAvailable: ContentGraph = new ContentGraph(); + assetGraph.traverse((node, context, actions) => { if (node.type !== 'asset') { return node; @@ -333,6 +337,41 @@ function createIdealGraph( dependencyPriorityEdges[dependency.priority], ); bundleGroupBundleIds.push(nodeId); + syncRootsAvailable.addNodeByContentKeyIfNeeded(asset.id, asset); + assetGraph.traverse((node, _, actions) => { + if (node.value === asset) { + return; + } + + if (node.type === 'dependency') { + let dependency = node.value; + + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return; + } + + invariant(assets.length === 1); + let resolved = assets[0]; + let isAsync = dependency.priority !== 'sync'; + if ( + isAsync || + resolved.bundleBehavior === 'isolated' || + resolved.bundleBehavior === 'inline' || + asset.type !== resolved.type + ) { + actions.skipChildren(); + } + + return; + } + //asset node type + let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( + node.value.id, + node.value, + ); + syncRootsAvailable.addEdge(rootNodeId, nodeId); + }, asset); } let assets = []; @@ -341,11 +380,18 @@ function createIdealGraph( Array<[Bundle, Asset]>, > = new DefaultMap(() => []); + let currentRoot; // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, // only adding the entry asset of each bundle, not the subgraph. assetGraph.traverse({ enter(node, context, actions) { if (node.type === 'asset') { + if (bundleRoots.has(node.value)) { + currentRoot = syncRootsAvailable.addNodeByContentKeyIfNeeded( + node.value.id, + node.value, + ); + } assets.push(node.value); let bundleIdTuple = bundleRoots.get(node.value); @@ -384,6 +430,36 @@ function createIdealGraph( bundleGraph.getNode(stack[0][1]), ); invariant(firstBundleGroup !== 'root'); + let entry = [...firstBundleGroup.assets][0]; + let entrynodeId = syncRootsAvailable.getNodeIdByContentKey( + entry.id, + ); + if (entrynodeId !== null) { + let hasThisAssetBySync = false; + syncRootsAvailable + .getNodeIdsConnectedFrom(entrynodeId) + .forEach(nodeId => { + let child = syncRootsAvailable.getNode(nodeId); + if (child?.id === childAsset.id) { + hasThisAssetBySync = true; + } + }); + if (hasThisAssetBySync) { + //for this dependency mark it internally resolvable for that bundle + let parentBundleId = bundles.has(parentAsset.id) + ? bundles.get(parentAsset.id) + : null; + let parentBundle; + if (parentBundleId !== null) { + parentBundle = nullthrows( + bundleGraph.getNode(parentBundleId), + ); + invariant(parentBundle !== 'root'); + parentBundle.internalizedAssetIds.push(childAsset.id); + } + continue; + } + } bundle = createBundle({ asset: childAsset, target: firstBundleGroup.target, @@ -398,6 +474,10 @@ function createIdealGraph( bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); + syncRootsAvailable.addNodeByContentKeyIfNeeded( + childAsset.id, + childAsset, + ); bundleGroupBundleIds.push(bundleId); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); } else { @@ -499,6 +579,10 @@ function createIdealGraph( bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); + syncRootsAvailable.addNodeByContentKeyIfNeeded( + childAsset.id, + childAsset, + ); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); if (bundleId != bundleGroupNodeId) { @@ -528,6 +612,13 @@ function createIdealGraph( assetReference.get(childAsset).push([dependency, bundle]); continue; } + if (dependency.priority === 'sync') { + let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( + childAsset.id, + childAsset, + ); + syncRootsAvailable.addEdge(currentRoot, nodeId); + } } } return node; @@ -787,12 +878,10 @@ function createIdealGraph( } } - function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { - let bundle = bundleGraph.getNode( - nullthrows(bundleRoots.get(bundleRoot))[0], - ); - invariant(bundle !== 'root' && bundle != null); - return bundle; + function getBundleFromBundleRoot(a: Asset): Bundle { + let b = nullthrows(bundleGraph.getNode(nullthrows(bundles.get(a.id)))); + invariant(b !== 'root'); + return b; } return { From d27025ea40f1f6f2340d644a08924a8f48d54f7f Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 11 Apr 2022 12:37:41 -0400 Subject: [PATCH 46/87] replace reachableroots with syncAssetsRequired and skip sync deps for entries later on --- .../experimental/src/ExperimentalBundler.js | 70 ++++++++----------- 1 file changed, 30 insertions(+), 40 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index abd1418f38f..41628fb6f05 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -291,7 +291,7 @@ function createIdealGraph( // Attempt to replace reachable roots by building it earlier. // this models bundleRoots and the assets that require it synchronously - let syncRootsAvailable: ContentGraph = new ContentGraph(); + let syncRequiredAssets: ContentGraph = new ContentGraph(); assetGraph.traverse((node, context, actions) => { if (node.type !== 'asset') { @@ -337,7 +337,7 @@ function createIdealGraph( dependencyPriorityEdges[dependency.priority], ); bundleGroupBundleIds.push(nodeId); - syncRootsAvailable.addNodeByContentKeyIfNeeded(asset.id, asset); + syncRequiredAssets.addNodeByContentKeyIfNeeded(asset.id, asset); assetGraph.traverse((node, _, actions) => { if (node.value === asset) { return; @@ -366,11 +366,11 @@ function createIdealGraph( return; } //asset node type - let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( + let nodeId = syncRequiredAssets.addNodeByContentKeyIfNeeded( node.value.id, node.value, ); - syncRootsAvailable.addEdge(rootNodeId, nodeId); + syncRequiredAssets.addEdge(rootNodeId, nodeId); }, asset); } @@ -430,24 +430,22 @@ function createIdealGraph( bundleGraph.getNode(stack[0][1]), ); invariant(firstBundleGroup !== 'root'); - let entry = [...firstBundleGroup.assets][0]; - let entrynodeId = syncRootsAvailable.getNodeIdByContentKey( + let entry = [...firstBundleGroup.assets][0]; //this may not be entry + let entrynodeId = syncRequiredAssets.getNodeIdByContentKey( entry.id, ); if (entrynodeId !== null) { - let hasThisAssetBySync = false; - syncRootsAvailable - .getNodeIdsConnectedFrom(entrynodeId) - .forEach(nodeId => { - let child = syncRootsAvailable.getNode(nodeId); - if (child?.id === childAsset.id) { - hasThisAssetBySync = true; - } - }); - if (hasThisAssetBySync) { + if ( + syncRequiredAssets + .getNodeIdsConnectedFrom(entrynodeId) + .some(nodeId => { + let child = syncRequiredAssets.getNode(nodeId); + return child?.id === childAsset.id; + }) + ) { //for this dependency mark it internally resolvable for that bundle let parentBundleId = bundles.has(parentAsset.id) - ? bundles.get(parentAsset.id) + ? nullthrows(bundles.get(parentAsset.id)) : null; let parentBundle; if (parentBundleId !== null) { @@ -474,10 +472,7 @@ function createIdealGraph( bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - syncRootsAvailable.addNodeByContentKeyIfNeeded( - childAsset.id, - childAsset, - ); + bundleGroupBundleIds.push(bundleId); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); } else { @@ -579,10 +574,7 @@ function createIdealGraph( bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - syncRootsAvailable.addNodeByContentKeyIfNeeded( - childAsset.id, - childAsset, - ); + bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); if (bundleId != bundleGroupNodeId) { @@ -612,13 +604,6 @@ function createIdealGraph( assetReference.get(childAsset).push([dependency, bundle]); continue; } - if (dependency.priority === 'sync') { - let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( - childAsset.id, - childAsset, - ); - syncRootsAvailable.addEdge(currentRoot, nodeId); - } } } return node; @@ -648,9 +633,11 @@ function createIdealGraph( } // Models bundleRoots and the assets that require it synchronously - let reachableRoots: ContentGraph = new ContentGraph(); for (let [root] of bundleRoots) { - let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); + let rootNodeId = syncRequiredAssets.addNodeByContentKeyIfNeeded( + root.id, + root, + ); assetGraph.traverse((node, _, actions) => { if (node.value === root) { return; @@ -704,12 +691,15 @@ function createIdealGraph( return; } + if (entries.has(root)) { + actions.skipChildren(); + } //asset node type - let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( + let nodeId = syncRequiredAssets.addNodeByContentKeyIfNeeded( node.value.id, node.value, ); - reachableRoots.addEdge(rootNodeId, nodeId); + syncRequiredAssets.addEdge(rootNodeId, nodeId); }, root); } @@ -752,11 +742,11 @@ function createIdealGraph( } let [siblingBundleRoot] = [...bundleInGroup.assets]; // Assets directly connected to current bundleRoot - let assetsFromBundleRoot = reachableRoots + let assetsFromBundleRoot = syncRequiredAssets .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(siblingBundleRoot.id), + syncRequiredAssets.getNodeIdByContentKey(siblingBundleRoot.id), ) - .map(id => nullthrows(reachableRoots.getNode(id))); + .map(id => nullthrows(syncRequiredAssets.getNode(id))); for (let asset of [siblingBundleRoot, ...assetsFromBundleRoot]) { available.add(asset); @@ -793,7 +783,7 @@ function createIdealGraph( // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means let reachable: Array = getReachableBundleRoots( asset, - reachableRoots, + syncRequiredAssets, ).reverse(); // Filter out bundles from this asset's reachable array if From 91170c3977d54ac49918fc9d178f457a11f99f48 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 26 Apr 2022 15:19:37 -0400 Subject: [PATCH 47/87] Revert "replace reachableroots with syncAssetsRequired and skip sync deps for entries later on" This reverts commit d27025ea40f1f6f2340d644a08924a8f48d54f7f. --- .../experimental/src/ExperimentalBundler.js | 70 +++++++++++-------- 1 file changed, 40 insertions(+), 30 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 41628fb6f05..abd1418f38f 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -291,7 +291,7 @@ function createIdealGraph( // Attempt to replace reachable roots by building it earlier. // this models bundleRoots and the assets that require it synchronously - let syncRequiredAssets: ContentGraph = new ContentGraph(); + let syncRootsAvailable: ContentGraph = new ContentGraph(); assetGraph.traverse((node, context, actions) => { if (node.type !== 'asset') { @@ -337,7 +337,7 @@ function createIdealGraph( dependencyPriorityEdges[dependency.priority], ); bundleGroupBundleIds.push(nodeId); - syncRequiredAssets.addNodeByContentKeyIfNeeded(asset.id, asset); + syncRootsAvailable.addNodeByContentKeyIfNeeded(asset.id, asset); assetGraph.traverse((node, _, actions) => { if (node.value === asset) { return; @@ -366,11 +366,11 @@ function createIdealGraph( return; } //asset node type - let nodeId = syncRequiredAssets.addNodeByContentKeyIfNeeded( + let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( node.value.id, node.value, ); - syncRequiredAssets.addEdge(rootNodeId, nodeId); + syncRootsAvailable.addEdge(rootNodeId, nodeId); }, asset); } @@ -430,22 +430,24 @@ function createIdealGraph( bundleGraph.getNode(stack[0][1]), ); invariant(firstBundleGroup !== 'root'); - let entry = [...firstBundleGroup.assets][0]; //this may not be entry - let entrynodeId = syncRequiredAssets.getNodeIdByContentKey( + let entry = [...firstBundleGroup.assets][0]; + let entrynodeId = syncRootsAvailable.getNodeIdByContentKey( entry.id, ); if (entrynodeId !== null) { - if ( - syncRequiredAssets - .getNodeIdsConnectedFrom(entrynodeId) - .some(nodeId => { - let child = syncRequiredAssets.getNode(nodeId); - return child?.id === childAsset.id; - }) - ) { + let hasThisAssetBySync = false; + syncRootsAvailable + .getNodeIdsConnectedFrom(entrynodeId) + .forEach(nodeId => { + let child = syncRootsAvailable.getNode(nodeId); + if (child?.id === childAsset.id) { + hasThisAssetBySync = true; + } + }); + if (hasThisAssetBySync) { //for this dependency mark it internally resolvable for that bundle let parentBundleId = bundles.has(parentAsset.id) - ? nullthrows(bundles.get(parentAsset.id)) + ? bundles.get(parentAsset.id) : null; let parentBundle; if (parentBundleId !== null) { @@ -472,7 +474,10 @@ function createIdealGraph( bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - + syncRootsAvailable.addNodeByContentKeyIfNeeded( + childAsset.id, + childAsset, + ); bundleGroupBundleIds.push(bundleId); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); } else { @@ -574,7 +579,10 @@ function createIdealGraph( bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - + syncRootsAvailable.addNodeByContentKeyIfNeeded( + childAsset.id, + childAsset, + ); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); if (bundleId != bundleGroupNodeId) { @@ -604,6 +612,13 @@ function createIdealGraph( assetReference.get(childAsset).push([dependency, bundle]); continue; } + if (dependency.priority === 'sync') { + let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( + childAsset.id, + childAsset, + ); + syncRootsAvailable.addEdge(currentRoot, nodeId); + } } } return node; @@ -633,11 +648,9 @@ function createIdealGraph( } // Models bundleRoots and the assets that require it synchronously + let reachableRoots: ContentGraph = new ContentGraph(); for (let [root] of bundleRoots) { - let rootNodeId = syncRequiredAssets.addNodeByContentKeyIfNeeded( - root.id, - root, - ); + let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); assetGraph.traverse((node, _, actions) => { if (node.value === root) { return; @@ -691,15 +704,12 @@ function createIdealGraph( return; } - if (entries.has(root)) { - actions.skipChildren(); - } //asset node type - let nodeId = syncRequiredAssets.addNodeByContentKeyIfNeeded( + let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( node.value.id, node.value, ); - syncRequiredAssets.addEdge(rootNodeId, nodeId); + reachableRoots.addEdge(rootNodeId, nodeId); }, root); } @@ -742,11 +752,11 @@ function createIdealGraph( } let [siblingBundleRoot] = [...bundleInGroup.assets]; // Assets directly connected to current bundleRoot - let assetsFromBundleRoot = syncRequiredAssets + let assetsFromBundleRoot = reachableRoots .getNodeIdsConnectedFrom( - syncRequiredAssets.getNodeIdByContentKey(siblingBundleRoot.id), + reachableRoots.getNodeIdByContentKey(siblingBundleRoot.id), ) - .map(id => nullthrows(syncRequiredAssets.getNode(id))); + .map(id => nullthrows(reachableRoots.getNode(id))); for (let asset of [siblingBundleRoot, ...assetsFromBundleRoot]) { available.add(asset); @@ -783,7 +793,7 @@ function createIdealGraph( // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means let reachable: Array = getReachableBundleRoots( asset, - syncRequiredAssets, + reachableRoots, ).reverse(); // Filter out bundles from this asset's reachable array if From 364f0819e1b816b55b936d7db9201d49aa3f379c Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 26 Apr 2022 15:20:04 -0400 Subject: [PATCH 48/87] Revert "move reachable root creation earlier to prevent unnecessary async bundle" This reverts commit 403ae53b20ca7175e98802dbc7f369fb49aa0b2d. --- .../experimental/src/ExperimentalBundler.js | 101 ++---------------- 1 file changed, 6 insertions(+), 95 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index abd1418f38f..e86da78f12b 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -289,10 +289,6 @@ function createIdealGraph( let entries: Map = new Map(); let sharedToSourceBundleIds: Map> = new Map(); - // Attempt to replace reachable roots by building it earlier. - // this models bundleRoots and the assets that require it synchronously - let syncRootsAvailable: ContentGraph = new ContentGraph(); - assetGraph.traverse((node, context, actions) => { if (node.type !== 'asset') { return node; @@ -337,41 +333,6 @@ function createIdealGraph( dependencyPriorityEdges[dependency.priority], ); bundleGroupBundleIds.push(nodeId); - syncRootsAvailable.addNodeByContentKeyIfNeeded(asset.id, asset); - assetGraph.traverse((node, _, actions) => { - if (node.value === asset) { - return; - } - - if (node.type === 'dependency') { - let dependency = node.value; - - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return; - } - - invariant(assets.length === 1); - let resolved = assets[0]; - let isAsync = dependency.priority !== 'sync'; - if ( - isAsync || - resolved.bundleBehavior === 'isolated' || - resolved.bundleBehavior === 'inline' || - asset.type !== resolved.type - ) { - actions.skipChildren(); - } - - return; - } - //asset node type - let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( - node.value.id, - node.value, - ); - syncRootsAvailable.addEdge(rootNodeId, nodeId); - }, asset); } let assets = []; @@ -380,18 +341,11 @@ function createIdealGraph( Array<[Bundle, Asset]>, > = new DefaultMap(() => []); - let currentRoot; // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, // only adding the entry asset of each bundle, not the subgraph. assetGraph.traverse({ enter(node, context, actions) { if (node.type === 'asset') { - if (bundleRoots.has(node.value)) { - currentRoot = syncRootsAvailable.addNodeByContentKeyIfNeeded( - node.value.id, - node.value, - ); - } assets.push(node.value); let bundleIdTuple = bundleRoots.get(node.value); @@ -430,36 +384,6 @@ function createIdealGraph( bundleGraph.getNode(stack[0][1]), ); invariant(firstBundleGroup !== 'root'); - let entry = [...firstBundleGroup.assets][0]; - let entrynodeId = syncRootsAvailable.getNodeIdByContentKey( - entry.id, - ); - if (entrynodeId !== null) { - let hasThisAssetBySync = false; - syncRootsAvailable - .getNodeIdsConnectedFrom(entrynodeId) - .forEach(nodeId => { - let child = syncRootsAvailable.getNode(nodeId); - if (child?.id === childAsset.id) { - hasThisAssetBySync = true; - } - }); - if (hasThisAssetBySync) { - //for this dependency mark it internally resolvable for that bundle - let parentBundleId = bundles.has(parentAsset.id) - ? bundles.get(parentAsset.id) - : null; - let parentBundle; - if (parentBundleId !== null) { - parentBundle = nullthrows( - bundleGraph.getNode(parentBundleId), - ); - invariant(parentBundle !== 'root'); - parentBundle.internalizedAssetIds.push(childAsset.id); - } - continue; - } - } bundle = createBundle({ asset: childAsset, target: firstBundleGroup.target, @@ -474,10 +398,6 @@ function createIdealGraph( bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - syncRootsAvailable.addNodeByContentKeyIfNeeded( - childAsset.id, - childAsset, - ); bundleGroupBundleIds.push(bundleId); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); } else { @@ -579,10 +499,6 @@ function createIdealGraph( bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - syncRootsAvailable.addNodeByContentKeyIfNeeded( - childAsset.id, - childAsset, - ); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); if (bundleId != bundleGroupNodeId) { @@ -612,13 +528,6 @@ function createIdealGraph( assetReference.get(childAsset).push([dependency, bundle]); continue; } - if (dependency.priority === 'sync') { - let nodeId = syncRootsAvailable.addNodeByContentKeyIfNeeded( - childAsset.id, - childAsset, - ); - syncRootsAvailable.addEdge(currentRoot, nodeId); - } } } return node; @@ -878,10 +787,12 @@ function createIdealGraph( } } - function getBundleFromBundleRoot(a: Asset): Bundle { - let b = nullthrows(bundleGraph.getNode(nullthrows(bundles.get(a.id)))); - invariant(b !== 'root'); - return b; + function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { + let bundle = bundleGraph.getNode( + nullthrows(bundleRoots.get(bundleRoot))[0], + ); + invariant(bundle !== 'root' && bundle != null); + return bundle; } return { From fb426b1b9dedc2d9c6692e7b6f11cfed6d550be2 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 26 Apr 2022 19:30:30 -0400 Subject: [PATCH 49/87] Implement cleanup for internalized deps from entrys --- .../experimental/src/ExperimentalBundler.js | 60 +++++++++++++++++++ .../core/integration-tests/test/javascript.js | 39 ++++++++++++ 2 files changed, 99 insertions(+) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e86da78f12b..a818aa0b456 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -732,6 +732,66 @@ function createIdealGraph( let bundleId = nullthrows(bundles.get(entry.id)); let bundle = nullthrows(bundleGraph.getNode(bundleId)); invariant(bundle !== 'root'); + let internalizedBundleId = bundles.get(asset.id); + if (internalizedBundleId !== undefined) { + let bundleRootsRequiring = nullthrows( + asyncBundleRootGraph.getNodeIdsConnectedTo( + asyncBundleRootGraph.getNodeIdByContentKey(asset.id), + ), + ); + for (let rootId of bundleRootsRequiring) { + let root = asyncBundleRootGraph.getNode(rootId); + if (root === 'root') continue; + let toBeInternalizedNodeId = + asyncBundleRootGraph.getNodeIdByContentKey(asset.id); + if (asyncAncestorAssets.get(root)?.has(asset)) { + let bundleId = bundles.get(root.id); + let b = bundleGraph.getNode(bundleId); + if (b && b !== 'root') { + b.internalizedAssetIds.push(asset.id); + } + + asyncBundleRootGraph.removeEdge(rootId, toBeInternalizedNodeId); + let remainingConnections = asyncBundleRootGraph.getNode( + toBeInternalizedNodeId, + ) + ? asyncBundleRootGraph + .getNodeIdsConnectedTo(toBeInternalizedNodeId) + .map(id => asyncBundleRootGraph.getNode(id)) + : []; + let containsRoot = remainingConnections.find( + node => node === 'root', + ); + if (remainingConnections.length === 1 && containsRoot) { + asyncBundleRootGraph.removeNode(toBeInternalizedNodeId); + } + if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { + bundleGraph.removeNode(bundles.get(asset.id)); + bundleRoots.delete(asset); + } + } else if (root === entry) { + let hasSync = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(entry.id), + ) + .some(id => { + let a = reachableRoots.getNode(id); + return a === asset; + }); + if (hasSync) { + bundle.internalizedAssetIds.push(asset.id); + asyncBundleRootGraph.removeEdge( + asyncBundleRootGraph.getNodeIdByContentKey(entry.id), + toBeInternalizedNodeId, + ); + } + if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { + bundleRoots.delete(asset); + bundleGraph.removeNode(bundles.get(asset.id)); + } + } + } + } bundle.assets.add(asset); bundle.size += asset.stats.size; } diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index f6bb4f9426a..d4cee83dbe3 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -4578,6 +4578,45 @@ describe('javascript', function () { assert.deepEqual(await (await run(b)).default, [42, 42, 42]); }); + it('async dependency can be resolved internally and externally from two different bundles', async () => { + let b = await bundle( + ['entry1.js', 'entry2.js'].map(entry => + path.join( + __dirname, + '/integration/async-dep-internal-external/', + entry, + ), + ), + { + mode: 'production', + defaultTargetOptions: { + shouldScopeHoist: true, + }, + }, + ); + + assertBundles(b, [ + { + assets: ['async.js'], + }, + { + name: 'entry1.js', + assets: ['child.js', 'entry1.js', 'async.js'], + }, + { + name: 'entry2.js', + assets: [ + 'bundle-manifest.js', + 'bundle-url.js', + 'cacheLoader.js', + 'child.js', + 'entry2.js', + 'js-loader.js', + ], + }, + ]); + }); + it('can static import and dynamic import in the same bundle ancestry without creating a new bundle', async () => { let b = await bundle( path.join(__dirname, '/integration/sync-async/same-ancestry.js'), From 30de01d011a3f2b769687d7cc9726130f66b47c6 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 26 Apr 2022 20:07:10 -0400 Subject: [PATCH 50/87] flow and clean up and comments --- .../experimental/src/ExperimentalBundler.js | 47 +++++++++++-------- 1 file changed, 27 insertions(+), 20 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index a818aa0b456..fd15fb3fdf1 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -729,29 +729,35 @@ function createIdealGraph( // Add assets to non-splittable bundles. for (let entry of reachableEntries) { - let bundleId = nullthrows(bundles.get(entry.id)); - let bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); + let entryBundleId = nullthrows(bundles.get(entry.id)); + let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); + invariant(entryBundle !== 'root'); + + // If this asset is associated with a bundle, we must attempt to clean up let internalizedBundleId = bundles.get(asset.id); if (internalizedBundleId !== undefined) { + let toBeInternalizedNodeId = + asyncBundleRootGraph.getNodeIdByContentKey(asset.id); let bundleRootsRequiring = nullthrows( - asyncBundleRootGraph.getNodeIdsConnectedTo( - asyncBundleRootGraph.getNodeIdByContentKey(asset.id), - ), + asyncBundleRootGraph.getNodeIdsConnectedTo(toBeInternalizedNodeId), ); - for (let rootId of bundleRootsRequiring) { - let root = asyncBundleRootGraph.getNode(rootId); - if (root === 'root') continue; - let toBeInternalizedNodeId = - asyncBundleRootGraph.getNodeIdByContentKey(asset.id); - if (asyncAncestorAssets.get(root)?.has(asset)) { - let bundleId = bundles.get(root.id); + //attempt to clean up for all immediate bundle connections + for (let parentBundleId of bundleRootsRequiring) { + let parentBundleRoot = asyncBundleRootGraph.getNode(parentBundleId); + if (parentBundleRoot === 'root') continue; + invariant(parentBundleRoot != null); + // we may remove the bundle's connection to anything that already has it from previous ancestors, i.e. asyncAnc + if (asyncAncestorAssets.get(parentBundleRoot)?.has(asset)) { + let bundleId = nullthrows(bundles.get(parentBundleRoot.id)); let b = bundleGraph.getNode(bundleId); if (b && b !== 'root') { b.internalizedAssetIds.push(asset.id); } - asyncBundleRootGraph.removeEdge(rootId, toBeInternalizedNodeId); + asyncBundleRootGraph.removeEdge( + parentBundleId, + toBeInternalizedNodeId, + ); let remainingConnections = asyncBundleRootGraph.getNode( toBeInternalizedNodeId, ) @@ -766,10 +772,11 @@ function createIdealGraph( asyncBundleRootGraph.removeNode(toBeInternalizedNodeId); } if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { - bundleGraph.removeNode(bundles.get(asset.id)); + bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); bundleRoots.delete(asset); } - } else if (root === entry) { + } else if (parentBundleRoot === entry) { + //entries do not have asyncAncestors so we must check if its available sync let hasSync = reachableRoots .getNodeIdsConnectedFrom( reachableRoots.getNodeIdByContentKey(entry.id), @@ -779,7 +786,7 @@ function createIdealGraph( return a === asset; }); if (hasSync) { - bundle.internalizedAssetIds.push(asset.id); + entryBundle.internalizedAssetIds.push(asset.id); asyncBundleRootGraph.removeEdge( asyncBundleRootGraph.getNodeIdByContentKey(entry.id), toBeInternalizedNodeId, @@ -787,13 +794,13 @@ function createIdealGraph( } if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { bundleRoots.delete(asset); - bundleGraph.removeNode(bundles.get(asset.id)); + bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); } } } } - bundle.assets.add(asset); - bundle.size += asset.stats.size; + entryBundle.assets.add(asset); + entryBundle.size += asset.stats.size; } // Create shared bundles for splittable bundles. From 239bb91bb57b6810ac5862159d7e994af5b1905d Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 27 Apr 2022 12:37:04 -0400 Subject: [PATCH 51/87] forgot to add the test files --- .../test/integration/async-dep-internal-external/async.js | 1 + .../test/integration/async-dep-internal-external/child.js | 1 + .../test/integration/async-dep-internal-external/entry1.js | 4 ++++ .../test/integration/async-dep-internal-external/entry2.js | 2 ++ 4 files changed, 8 insertions(+) create mode 100644 packages/core/integration-tests/test/integration/async-dep-internal-external/async.js create mode 100644 packages/core/integration-tests/test/integration/async-dep-internal-external/child.js create mode 100644 packages/core/integration-tests/test/integration/async-dep-internal-external/entry1.js create mode 100644 packages/core/integration-tests/test/integration/async-dep-internal-external/entry2.js diff --git a/packages/core/integration-tests/test/integration/async-dep-internal-external/async.js b/packages/core/integration-tests/test/integration/async-dep-internal-external/async.js new file mode 100644 index 00000000000..a4479d21f1f --- /dev/null +++ b/packages/core/integration-tests/test/integration/async-dep-internal-external/async.js @@ -0,0 +1 @@ +export default 30; diff --git a/packages/core/integration-tests/test/integration/async-dep-internal-external/child.js b/packages/core/integration-tests/test/integration/async-dep-internal-external/child.js new file mode 100644 index 00000000000..17f1e128990 --- /dev/null +++ b/packages/core/integration-tests/test/integration/async-dep-internal-external/child.js @@ -0,0 +1 @@ +export default import('./async').then(mod => mod.default); diff --git a/packages/core/integration-tests/test/integration/async-dep-internal-external/entry1.js b/packages/core/integration-tests/test/integration/async-dep-internal-external/entry1.js new file mode 100644 index 00000000000..04a92630d9e --- /dev/null +++ b/packages/core/integration-tests/test/integration/async-dep-internal-external/entry1.js @@ -0,0 +1,4 @@ +import a from './async'; +import c from './child'; + +output = [a,c]; diff --git a/packages/core/integration-tests/test/integration/async-dep-internal-external/entry2.js b/packages/core/integration-tests/test/integration/async-dep-internal-external/entry2.js new file mode 100644 index 00000000000..a0ddf7190a1 --- /dev/null +++ b/packages/core/integration-tests/test/integration/async-dep-internal-external/entry2.js @@ -0,0 +1,2 @@ +import c from './child'; + From 137801ca60e8da7ade25aad6fdf168800d56fb80 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 27 Apr 2022 16:25:56 -0400 Subject: [PATCH 52/87] skip assets for reachable if isolated or inline fix invariant --- .../experimental/src/ExperimentalBundler.js | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index fd15fb3fdf1..099a300954a 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -594,26 +594,22 @@ function createIdealGraph( } } - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return; - } - - invariant(assets.length === 1); - let resolved = assets[0]; - let isAsync = dependency.priority !== 'sync'; - if ( - isAsync || - resolved.bundleBehavior === 'isolated' || - resolved.bundleBehavior === 'inline' || - root.type !== resolved.type - ) { + if (dependency.priority !== 'sync') { actions.skipChildren(); } return; } //asset node type + let asset = node.value; + if ( + asset.bundleBehavior === 'isolated' || + asset.bundleBehavior === 'inline' || + root.type !== asset.type + ) { + actions.skipChildren(); + return; + } let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( node.value.id, node.value, From e87347d9e17d296d3d44363fc66398c68f9aadf7 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 27 Apr 2022 16:25:56 -0400 Subject: [PATCH 53/87] skip assets for reachable if isolated or inline fix invariant --- .../experimental/src/ExperimentalBundler.js | 83 +++++++++---------- 1 file changed, 41 insertions(+), 42 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e45c7c07777..33db61a75e8 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -564,56 +564,55 @@ function createIdealGraph( if (node.value === root) { return; } + if (node.type === 'dependency') { + let dependency = node.value; - if (node.type === 'asset') { - let asset = node.value; - if ( - asset.bundleBehavior === 'isolated' || - asset.bundleBehavior === 'inline' || - root.type !== asset.type - ) { - actions.skipChildren(); - return; - } - - let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( - node.value.id, - node.value, - ); - reachableRoots.addEdge(rootNodeId, nodeId); - return; - } - - let dependency = node.value; - if (dependencyBundleGraph.hasContentKey(dependency.id)) { - if (dependency.priority !== 'sync') { - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return; - } + if (dependencyBundleGraph.hasContentKey(dependency.id)) { + if (dependency.priority !== 'sync') { + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return; + } - invariant(assets.length === 1); - let bundleRoot = assets[0]; - let bundle = nullthrows( - bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), - ); - if ( - bundle !== 'root' && - bundle.bundleBehavior !== 'isolated' && - bundle.bundleBehavior !== 'inline' && - !bundle.env.isIsolated() - ) { - asyncBundleRootGraph.addEdge( - asyncBundleRootGraph.getNodeIdByContentKey(root.id), - asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + invariant(assets.length === 1); + let bundleRoot = assets[0]; + let bundle = nullthrows( + bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), ); + if ( + bundle !== 'root' && + bundle.bundleBehavior !== 'isolated' && + bundle.bundleBehavior !== 'inline' && + !bundle.env.isIsolated() + ) { + asyncBundleRootGraph.addEdge( + asyncBundleRootGraph.getNodeIdByContentKey(root.id), + asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + ); + } } } - } - if (dependency.priority !== 'sync') { + if (dependency.priority !== 'sync') { + actions.skipChildren(); + } + return; + } + //asset node type + let asset = node.value; + if ( + asset.bundleBehavior === 'isolated' || + asset.bundleBehavior === 'inline' || + root.type !== asset.type + ) { actions.skipChildren(); + return; } + let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( + node.value.id, + node.value, + ); + reachableRoots.addEdge(rootNodeId, nodeId); }, root); } From e5c4c8a1f06ea344ac77d40983489557c1a5f1e0 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Fri, 29 Apr 2022 11:26:00 -0700 Subject: [PATCH 54/87] Use bundleGroup instead of bundle root for determining needsStableName --- .../bundlers/experimental/src/ExperimentalBundler.js | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 33db61a75e8..8621be7f72c 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -426,9 +426,7 @@ function createIdealGraph( dependency.priority === 'parallel' || childAsset.bundleBehavior === 'inline' ) { - let [parentBundleRoot, bundleGroupNodeId] = nullthrows( - stack[stack.length - 1], - ); + let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); let bundleGroup = nullthrows( bundleGraph.getNode(bundleGroupNodeId), ); @@ -461,12 +459,6 @@ function createIdealGraph( let bundle; if (bundleId == null) { - let parentBundleId = nullthrows(bundles.get(parentBundleRoot.id)); - let parentBundle = nullthrows( - bundleGraph.getNode(parentBundleId), - ); - invariant(parentBundle !== 'root'); - // Create a new bundle if none of the same type exists already. bundle = createBundle({ // We either have an entry asset or a unique key. @@ -483,7 +475,7 @@ function createIdealGraph( (dependency.priority === 'parallel' && !dependency.needsStableName) ? false - : parentBundle.needsStableName, + : bundleGroup.needsStableName, }); bundleId = bundleGraph.addNode(bundle); } else { From 7faa3d738f17d1a710963fd0e6f47cd9d0959a77 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Mon, 2 May 2022 11:08:41 -0700 Subject: [PATCH 55/87] remove asset references for deleted bundles + minor fixes --- .../experimental/src/ExperimentalBundler.js | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index cf45f4220fc..7b127ef846c 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -878,7 +878,7 @@ function createIdealGraph( if (bundle === 'root') continue; if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { sharedToSourceBundleIds.delete(bundleNodeId); - removeBundle(bundleGraph, bundleNodeId); + removeBundle(bundleGraph, bundleNodeId, assetReference); } } @@ -931,7 +931,7 @@ function createIdealGraph( let incomingNodeCount = bundleGraph.getNodeIdsConnectedTo(bundleId).length; if (incomingNodeCount === 1) { - removeBundle(bundleGraph, bundleId); + removeBundle(bundleGraph, bundleId, assetReference); } else if (incomingNodeCount === 0) { bundleGraph.removeNode(bundleId); } @@ -1007,11 +1007,18 @@ function createBundle(opts: {| }; } -function removeBundle(bundleGraph: Graph, bundleId: NodeId) { +function removeBundle( + bundleGraph: Graph, + bundleId: NodeId, + assetReference: DefaultMap>, +) { let bundle = nullthrows(bundleGraph.getNode(bundleId)); invariant(bundle !== 'root'); - for (let asset of bundle.assets) { + assetReference.set( + asset, + assetReference.get(asset).filter(t => !t.includes(bundle)), + ); for (let sourceBundleId of bundle.sourceBundles) { let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); invariant(sourceBundle !== 'root'); @@ -1028,7 +1035,7 @@ async function loadBundlerConfig( options: PluginOptions, ): Promise { let conf = await config.getConfig([], { - packageKey: '@parcel/bundler-default', + packageKey: '@parcel/bundler-experimental', }); if (!conf) { return HTTP_OPTIONS['2']; @@ -1042,10 +1049,10 @@ async function loadBundlerConfig( data: conf?.contents, source: await options.inputFS.readFile(conf.filePath, 'utf8'), filePath: conf.filePath, - prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-default')}`, + prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-experimental')}`, }, - '@parcel/bundler-default', - 'Invalid config for @parcel/bundler-default', + '@parcel/bundler-experimental', + 'Invalid config for @parcel/bundler-experimental', ); let http = conf.contents.http ?? 2; From 955fb67ccdd8116db5bbbb521420d9edbb34fd6a Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 4 May 2022 14:25:53 -0400 Subject: [PATCH 56/87] Filter out bundleroots from reacable if they are subgraphs, consider isIsolated --- .../experimental/src/ExperimentalBundler.js | 263 ++++++++++-------- 1 file changed, 145 insertions(+), 118 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 099a300954a..31978287341 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -705,138 +705,165 @@ function createIdealGraph( // bundle does not contain the asset in its ancestry reachable = reachable.filter(b => !asyncAncestorAssets.get(b)?.has(asset)); - if (reachable.length > 0) { - let reachableEntries = reachable.filter( - a => - entries.has(a) || - !a.isBundleSplittable || - getBundleFromBundleRoot(a).needsStableName || - getBundleFromBundleRoot(a).bundleBehavior === 'inline' || - getBundleFromBundleRoot(a).bundleBehavior === 'isolated', - ); - reachable = reachable.filter( - a => - !entries.has(a) && - a.isBundleSplittable && - !getBundleFromBundleRoot(a).needsStableName && - getBundleFromBundleRoot(a).bundleBehavior !== 'inline' && - getBundleFromBundleRoot(a).bundleBehavior !== 'isolated', - ); + let reachableEntries = reachable.filter( + a => + entries.has(a) || + !a.isBundleSplittable || + getBundleFromBundleRoot(a).needsStableName || + getBundleFromBundleRoot(a).bundleBehavior === 'inline' || + getBundleFromBundleRoot(a).bundleBehavior === 'isolated', + ); + reachable = reachable.filter( + a => + !entries.has(a) && + a.isBundleSplittable && + !getBundleFromBundleRoot(a).needsStableName && + getBundleFromBundleRoot(a).bundleBehavior !== 'inline' && + getBundleFromBundleRoot(a).bundleBehavior !== 'isolated', + ); - // Add assets to non-splittable bundles. - for (let entry of reachableEntries) { - let entryBundleId = nullthrows(bundles.get(entry.id)); - let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); - invariant(entryBundle !== 'root'); - - // If this asset is associated with a bundle, we must attempt to clean up - let internalizedBundleId = bundles.get(asset.id); - if (internalizedBundleId !== undefined) { - let toBeInternalizedNodeId = - asyncBundleRootGraph.getNodeIdByContentKey(asset.id); - let bundleRootsRequiring = nullthrows( - asyncBundleRootGraph.getNodeIdsConnectedTo(toBeInternalizedNodeId), + reachable = reachable.filter(b => { + if (b.env.isIsolated()) { + return true; + } + let toKeep = true; + if (bundles.has(asset.id)) { + toKeep = false; + bundleGraph.addEdge( + nullthrows(bundles.get(b.id)), + nullthrows(bundles.get(asset.id)), + ); + } + for (let f of reachable) { + if (b === f) continue; + let fReachable = getReachableBundleRoots(f, reachableRoots).filter( + b => !asyncAncestorAssets.get(b)?.has(f), + ); + if (fReachable.indexOf(b) > -1) { + toKeep = false; + bundleGraph.addEdge( + nullthrows(bundles.get(b.id)), + nullthrows(bundles.get(f.id)), ); - //attempt to clean up for all immediate bundle connections - for (let parentBundleId of bundleRootsRequiring) { - let parentBundleRoot = asyncBundleRootGraph.getNode(parentBundleId); - if (parentBundleRoot === 'root') continue; - invariant(parentBundleRoot != null); - // we may remove the bundle's connection to anything that already has it from previous ancestors, i.e. asyncAnc - if (asyncAncestorAssets.get(parentBundleRoot)?.has(asset)) { - let bundleId = nullthrows(bundles.get(parentBundleRoot.id)); - let b = bundleGraph.getNode(bundleId); - if (b && b !== 'root') { - b.internalizedAssetIds.push(asset.id); - } + } + } + return toKeep; + }); + + // Add assets to non-splittable bundles. + for (let entry of reachableEntries) { + let entryBundleId = nullthrows(bundles.get(entry.id)); + let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); + invariant(entryBundle !== 'root'); + + // If this asset is associated with a bundle, we must attempt to clean up + let internalizedBundleId = bundles.get(asset.id); + if (internalizedBundleId !== undefined) { + let toBeInternalizedNodeId = asyncBundleRootGraph.getNodeIdByContentKey( + asset.id, + ); + let bundleRootsRequiring = nullthrows( + asyncBundleRootGraph.getNodeIdsConnectedTo(toBeInternalizedNodeId), + ); + //attempt to clean up for all immediate bundle connections + for (let parentBundleId of bundleRootsRequiring) { + let parentBundleRoot = asyncBundleRootGraph.getNode(parentBundleId); + if (parentBundleRoot === 'root') continue; + invariant(parentBundleRoot != null); + // we may remove the bundle's connection to anything that already has it from previous ancestors, i.e. asyncAnc + if (asyncAncestorAssets.get(parentBundleRoot)?.has(asset)) { + let bundleId = nullthrows(bundles.get(parentBundleRoot.id)); + let b = bundleGraph.getNode(bundleId); + if (b && b !== 'root') { + b.internalizedAssetIds.push(asset.id); + } + asyncBundleRootGraph.removeEdge( + parentBundleId, + toBeInternalizedNodeId, + ); + let remainingConnections = asyncBundleRootGraph.getNode( + toBeInternalizedNodeId, + ) + ? asyncBundleRootGraph + .getNodeIdsConnectedTo(toBeInternalizedNodeId) + .map(id => asyncBundleRootGraph.getNode(id)) + : []; + let containsRoot = remainingConnections.find( + node => node === 'root', + ); + if (remainingConnections.length === 1 && containsRoot) { + asyncBundleRootGraph.removeNode(toBeInternalizedNodeId); + } + if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { + bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); + bundleRoots.delete(asset); + } + } else if (parentBundleRoot === entry) { + //entries do not have asyncAncestors so we must check if its available sync + let hasSync = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(entry.id), + ) + .some(id => { + let a = reachableRoots.getNode(id); + return a === asset; + }); + if (hasSync) { + entryBundle.internalizedAssetIds.push(asset.id); asyncBundleRootGraph.removeEdge( - parentBundleId, + asyncBundleRootGraph.getNodeIdByContentKey(entry.id), toBeInternalizedNodeId, ); - let remainingConnections = asyncBundleRootGraph.getNode( - toBeInternalizedNodeId, - ) - ? asyncBundleRootGraph - .getNodeIdsConnectedTo(toBeInternalizedNodeId) - .map(id => asyncBundleRootGraph.getNode(id)) - : []; - let containsRoot = remainingConnections.find( - node => node === 'root', - ); - if (remainingConnections.length === 1 && containsRoot) { - asyncBundleRootGraph.removeNode(toBeInternalizedNodeId); - } - if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { - bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); - bundleRoots.delete(asset); - } - } else if (parentBundleRoot === entry) { - //entries do not have asyncAncestors so we must check if its available sync - let hasSync = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(entry.id), - ) - .some(id => { - let a = reachableRoots.getNode(id); - return a === asset; - }); - if (hasSync) { - entryBundle.internalizedAssetIds.push(asset.id); - asyncBundleRootGraph.removeEdge( - asyncBundleRootGraph.getNodeIdByContentKey(entry.id), - toBeInternalizedNodeId, - ); - } - if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { - bundleRoots.delete(asset); - bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); - } + } + if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { + bundleRoots.delete(asset); + bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); } } } - entryBundle.assets.add(asset); - entryBundle.size += asset.stats.size; } + entryBundle.assets.add(asset); + entryBundle.size += asset.stats.size; + } - // Create shared bundles for splittable bundles. - if (reachable.length > 0) { - let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); - let key = reachable.map(a => a.id).join(','); - let bundleId = bundles.get(key); - let bundle; - if (bundleId == null) { - let firstSourceBundle = nullthrows( - bundleGraph.getNode(sourceBundles[0]), - ); - invariant(firstSourceBundle !== 'root'); - bundle = createBundle({ - target: firstSourceBundle.target, - type: firstSourceBundle.type, - env: firstSourceBundle.env, - }); - bundle.sourceBundles = sourceBundles; - bundleId = bundleGraph.addNode(bundle); - bundles.set(key, bundleId); - } else { - bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); - } - bundle.assets.add(asset); - bundle.size += asset.stats.size; + // Create shared bundles for splittable bundles. + if (reachable.length > 0) { + let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); + let key = reachable.map(a => a.id).join(','); + let bundleId = bundles.get(key); + let bundle; + if (bundleId == null) { + let firstSourceBundle = nullthrows( + bundleGraph.getNode(sourceBundles[0]), + ); + invariant(firstSourceBundle !== 'root'); + bundle = createBundle({ + target: firstSourceBundle.target, + type: firstSourceBundle.type, + env: firstSourceBundle.env, + }); + bundle.sourceBundles = sourceBundles; + bundleId = bundleGraph.addNode(bundle); + bundles.set(key, bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); + } + bundle.assets.add(asset); + bundle.size += asset.stats.size; - for (let sourceBundleId of sourceBundles) { - if (bundleId !== sourceBundleId) { - bundleGraph.addEdge(sourceBundleId, bundleId); - } + for (let sourceBundleId of sourceBundles) { + if (bundleId !== sourceBundleId) { + bundleGraph.addEdge(sourceBundleId, bundleId); } - sharedToSourceBundleIds.set(bundleId, sourceBundles); - - dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { - value: bundle, - type: 'bundle', - }); } + sharedToSourceBundleIds.set(bundleId, sourceBundles); + + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { + value: bundle, + type: 'bundle', + }); } } From fe4efb4d36791c4abee4d4bfc24dff10a95391c2 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Fri, 29 Apr 2022 12:37:21 -0700 Subject: [PATCH 57/87] Add bundle.mainEntryAsset --- .../experimental/src/ExperimentalBundler.js | 27 +++++++++++-------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 8621be7f72c..42698fa87d1 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -58,6 +58,7 @@ export type Bundle = {| internalizedAssetIds: Array, bundleBehavior?: ?BundleBehavior, needsStableName: boolean, + mainEntryAsset: ?Asset, size: number, sourceBundles: Array, target: Target, @@ -118,7 +119,7 @@ function decorateLegacyGraph( // Step 1: Create bundle groups, bundles, and shared bundles and add assets to them for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { if (idealBundle === 'root') continue; - let [entryAsset] = [...idealBundle.assets]; + let entryAsset = idealBundle.mainEntryAsset; let bundleGroup; let bundle; @@ -645,16 +646,18 @@ function createIdealGraph( ) { continue; } - let [siblingBundleRoot] = [...bundleInGroup.assets]; - // Assets directly connected to current bundleRoot - let assetsFromBundleRoot = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(siblingBundleRoot.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); - - for (let asset of [siblingBundleRoot, ...assetsFromBundleRoot]) { - available.add(asset); + + for (let bundleRoot of bundleInGroup.assets) { + // Assets directly connected to current bundleRoot + let assetsFromBundleRoot = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); + + for (let asset of [bundleRoot, ...assetsFromBundleRoot]) { + available.add(asset); + } } } } @@ -824,6 +827,7 @@ function createBundle(opts: {| uniqueKey: opts.uniqueKey, assets: new Set(), internalizedAssetIds: [], + mainEntryAsset: null, size: 0, sourceBundles: [], target: opts.target, @@ -839,6 +843,7 @@ function createBundle(opts: {| uniqueKey: opts.uniqueKey, assets: new Set([asset]), internalizedAssetIds: [], + mainEntryAsset: asset, size: asset.stats.size, sourceBundles: [], target: opts.target, From 95b85889ff1d9ea182fe6cd631d9d9b7c8034d52 Mon Sep 17 00:00:00 2001 From: Will Binns-Smith Date: Tue, 3 May 2022 15:55:28 -0700 Subject: [PATCH 58/87] ExperimentalBundler: merge bundleBehavior and needsStableName --- .../experimental/src/ExperimentalBundler.js | 9 ++ .../core/integration-tests/test/javascript.js | 105 ++++++++++++------ 2 files changed, 79 insertions(+), 35 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 42698fa87d1..805d7456cd1 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -404,6 +404,15 @@ function createIdealGraph( } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); invariant(bundle !== 'root'); + + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } } dependencyBundleGraph.addEdge( diff --git a/packages/core/integration-tests/test/javascript.js b/packages/core/integration-tests/test/javascript.js index 8a77a27394c..1b0e229bd9d 100644 --- a/packages/core/integration-tests/test/javascript.js +++ b/packages/core/integration-tests/test/javascript.js @@ -5699,27 +5699,47 @@ describe('javascript', function () { ), ); - assertBundles(b, [ - { - type: 'js', - assets: [ - 'dynamic-url.js', - 'esmodule-helpers.js', - 'bundle-url.js', - 'cacheLoader.js', - 'js-loader.js', - ], - }, - { - type: 'js', - assets: ['other.js'], - }, - { - type: 'js', - assets: ['other.js', 'esmodule-helpers.js'], - }, - ]); - + // Change in behavior: ExperimentalBundler now produces a single bundle + // of the lowest common denominator of bundleBehavior + if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { + assertBundles(b, [ + { + type: 'js', + assets: [ + 'dynamic-url.js', + 'esmodule-helpers.js', + 'bundle-url.js', + 'cacheLoader.js', + 'js-loader.js', + ], + }, + { + type: 'js', + assets: ['other.js', 'esmodule-helpers.js'], + }, + ]); + } else { + assertBundles(b, [ + { + type: 'js', + assets: [ + 'dynamic-url.js', + 'esmodule-helpers.js', + 'bundle-url.js', + 'cacheLoader.js', + 'js-loader.js', + ], + }, + { + type: 'js', + assets: ['other.js'], + }, + { + type: 'js', + assets: ['other.js', 'esmodule-helpers.js'], + }, + ]); + } let res = await run(b); assert.equal(typeof res.lazy, 'object'); assert.equal(typeof (await res.lazy), 'function'); @@ -5744,20 +5764,35 @@ describe('javascript', function () { }, ); - assertBundles(b, [ - { - type: 'js', - assets: ['dynamic-url.js'], - }, - { - type: 'js', - assets: ['other.js'], - }, - { - type: 'js', - assets: ['other.js'], - }, - ]); + if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { + // Change in behavior: ExperimentalBundler now produces a single bundle + // of the lowest common denominator of bundleBehavior + assertBundles(b, [ + { + type: 'js', + assets: ['dynamic-url.js'], + }, + { + type: 'js', + assets: ['other.js'], + }, + ]); + } else { + assertBundles(b, [ + { + type: 'js', + assets: ['dynamic-url.js'], + }, + { + type: 'js', + assets: ['other.js'], + }, + { + type: 'js', + assets: ['other.js'], + }, + ]); + } let res = await run(b); assert.equal(typeof res.lazy, 'object'); From 59f4ef4e31a50b9c2a2c21812b2ab9d2f29a3597 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 4 May 2022 16:37:53 -0400 Subject: [PATCH 59/87] don't remove bundles depended on by url --- .../experimental/src/ExperimentalBundler.js | 37 +++++++++++++------ packages/core/core/src/BundleGraph.js | 13 ++++--- 2 files changed, 32 insertions(+), 18 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 37b7476fe9c..e68e09dadd1 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -207,6 +207,7 @@ function decorateLegacyGraph( for (let incomingDep of incomingDeps) { if ( incomingDep.priority === 'lazy' && + incomingDep.specifierType !== 'url' && bundle.hasDependency(incomingDep) ) { bundleGraph.internalizeAsyncDependency(bundle, incomingDep); @@ -583,7 +584,6 @@ function createIdealGraph( ); if ( bundle !== 'root' && - bundle.bundleBehavior !== 'isolated' && bundle.bundleBehavior !== 'inline' && !bundle.env.isIsolated() ) { @@ -703,10 +703,6 @@ function createIdealGraph( reachableRoots, ).reverse(); - // Filter out bundles from this asset's reachable array if - // bundle does not contain the asset in its ancestry - reachable = reachable.filter(b => !asyncAncestorAssets.get(b)?.has(asset)); - let reachableEntries = reachable.filter( a => entries.has(a) || @@ -724,6 +720,10 @@ function createIdealGraph( getBundleFromBundleRoot(a).bundleBehavior !== 'isolated', ); + // Filter out bundles from this asset's reachable array if + // bundle does not contain the asset in its ancestry + reachable = reachable.filter(b => !asyncAncestorAssets.get(b)?.has(asset)); + reachable = reachable.filter(b => { if (b.env.isIsolated()) { return true; @@ -797,9 +797,12 @@ function createIdealGraph( if (remainingConnections.length === 1 && containsRoot) { asyncBundleRootGraph.removeNode(toBeInternalizedNodeId); } - if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { - bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); - bundleRoots.delete(asset); + if ( + !asyncBundleRootGraph.hasNode(toBeInternalizedNodeId) && + nullthrows(bundleGraph.getNode(toBeInternalizedNodeId)) + .bundleBehavior !== 'isolated' + ) { + deleteBundle(asset); } } else if (parentBundleRoot === entry) { //entries do not have asyncAncestors so we must check if its available sync @@ -818,9 +821,12 @@ function createIdealGraph( toBeInternalizedNodeId, ); } - if (!asyncBundleRootGraph.hasNode(toBeInternalizedNodeId)) { - bundleRoots.delete(asset); - bundleGraph.removeNode(nullthrows(bundles.get(asset.id))); + if ( + !asyncBundleRootGraph.hasNode(toBeInternalizedNodeId) && + nullthrows(bundleGraph.getNode(toBeInternalizedNodeId)) + .bundleBehavior !== 'isolated' + ) { + deleteBundle(asset); } } } @@ -935,7 +941,14 @@ function createIdealGraph( } } } - + function deleteBundle(bundleRoot: BundleRoot) { + bundleGraph.removeNode(nullthrows(bundles.get(bundleRoot.id))); + bundleRoots.delete(bundleRoot); + reachableRoots.replaceNodeIdsConnectedTo( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + [], + ); + } function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { let bundle = bundleGraph.getNode( nullthrows(bundleRoots.get(bundleRoot))[0], diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 4ee291557fd..215e64b0d6f 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -732,12 +732,13 @@ export default class BundleGraph { if ( inboundDependencies.every( dependency => - !this.bundleHasDependency(bundle, dependency) || - this._graph.hasEdge( - bundleNodeId, - this._graph.getNodeIdByContentKey(dependency.id), - bundleGraphEdgeTypes.internal_async, - ), + dependency.specifierType !== SpecifierType.url && + (!this.bundleHasDependency(bundle, dependency) || + this._graph.hasEdge( + bundleNodeId, + this._graph.getNodeIdByContentKey(dependency.id), + bundleGraphEdgeTypes.internal_async, + )), ) ) { this._graph.removeEdge( From c36daf02ded73690240dcfa6d61ac28a662b29ac Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 4 May 2022 17:41:31 -0400 Subject: [PATCH 60/87] don't flatted bundle to bundlegroup edges --- packages/bundlers/experimental/src/ExperimentalBundler.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index e68e09dadd1..f476c308082 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -353,7 +353,7 @@ function createIdealGraph( let bundleIdTuple = bundleRoots.get(node.value); if (bundleIdTuple) { // Push to the stack when a new bundle is created - stack.push([node.value, bundleIdTuple[1]]); + stack.push([node.value, bundleIdTuple[0]]); } } else if (node.type === 'dependency') { if (context == null) { From c7a80fa168eddd3528616243f9b69fda627cda3f Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Tue, 10 May 2022 17:08:57 -0400 Subject: [PATCH 61/87] point config to default bundler, crete type change bundle regardless them clean based on bundlegroups, add referencing bundle to stack when creating bundles instead of only parent or bundlegroup, add merge and delete bundles to clean all structures --- .../experimental/src/ExperimentalBundler.js | 208 ++++++++++++++---- packages/core/core/src/BundleGraph.js | 8 +- packages/core/integration-tests/test/html.js | 20 +- packages/core/utils/src/collection.js | 12 + packages/core/utils/src/index.js | 1 + 5 files changed, 190 insertions(+), 59 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index f476c308082..0cee3388f23 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -19,7 +19,12 @@ import {ContentGraph, Graph} from '@parcel/graph'; import invariant from 'assert'; import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; -import {setIntersect, validateSchema, DefaultMap} from '@parcel/utils'; +import { + setIntersect, + setEqual, + validateSchema, + DefaultMap, +} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -86,7 +91,7 @@ type DependencyBundleGraph = ContentGraph< type IdealGraph = {| dependencyBundleGraph: DependencyBundleGraph, bundleGraph: Graph, - bundleGroupBundleIds: Array, + bundleGroupBundleIds: Set, assetReference: DefaultMap>, sharedToSourceBundleIds: Map>, |}; @@ -123,7 +128,7 @@ function decorateLegacyGraph( let bundleGroup; let bundle; - if (bundleGroupBundleIds.includes(bundleNodeId)) { + if (bundleGroupBundleIds.has(bundleNodeId)) { let dependencies = dependencyBundleGraph .getNodeIdsConnectedTo( dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), @@ -285,7 +290,10 @@ function createIdealGraph( // bundleGraph that models bundleRoots and async deps only let asyncBundleRootGraph: ContentGraph = new ContentGraph(); - let bundleGroupBundleIds: Array = []; + let bundleGroupBundleIds: Set = new Set(); + + // Models bundleRoots and the assets that require it synchronously + let reachableRoots: ContentGraph = new ContentGraph(); // Step 1: Find and create bundles for entries from assetGraph let entries: Map = new Map(); @@ -334,7 +342,7 @@ function createIdealGraph( }), dependencyPriorityEdges[dependency.priority], ); - bundleGroupBundleIds.push(nodeId); + bundleGroupBundleIds.add(nodeId); } let assets = []; @@ -343,6 +351,7 @@ function createIdealGraph( Array<[Bundle, Asset]>, > = new DefaultMap(() => []); + let typeChangeIds = new Set(); // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, // only adding the entry asset of each bundle, not the subgraph. assetGraph.traverse({ @@ -351,9 +360,11 @@ function createIdealGraph( assets.push(node.value); let bundleIdTuple = bundleRoots.get(node.value); - if (bundleIdTuple) { + if (bundleIdTuple && bundleIdTuple[0] === bundleIdTuple[1]) { // Push to the stack when a new bundle is created stack.push([node.value, bundleIdTuple[0]]); + } else if (bundleIdTuple) { + stack.push([node.value, stack[stack.length - 1][1]]); } } else if (node.type === 'dependency') { if (context == null) { @@ -400,7 +411,7 @@ function createIdealGraph( bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - bundleGroupBundleIds.push(bundleId); + bundleGroupBundleIds.add(bundleId); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); @@ -437,7 +448,9 @@ function createIdealGraph( dependency.priority === 'parallel' || childAsset.bundleBehavior === 'inline' ) { - let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); + let [referencingBundleRoot, bundleGroupNodeId] = nullthrows( + stack[stack.length - 1], + ); let bundleGroup = nullthrows( bundleGraph.getNode(bundleGroupNodeId), ); @@ -447,28 +460,35 @@ function createIdealGraph( let bundleId; let entryAsset; let uniqueKey; - if ( - childAsset.bundleBehavior !== 'inline' && - dependency.priority !== 'parallel' - ) { - uniqueKey = childAsset.id; - // TODO: share bundles even across different bundle groups by looking if the child - // asset is already a bundle root. In order for this to work, bundleRoots must be - // keyed by asset + target, not just asset, so that bundles are not shared between targets. - bundleId = - bundleGroup.type == childAsset.type - ? bundleGroupNodeId - : bundleGraph - .getNodeIdsConnectedFrom(bundleGroupNodeId) - .find(id => { - let node = bundleGraph.getNode(id); - return node !== 'root' && node?.type == childAsset.type; - }); - } else { - entryAsset = childAsset; - } - + // if ( + // childAsset.bundleBehavior !== 'inline' && + // dependency.priority !== 'parallel' + // ) { + // uniqueKey = childAsset.id; + // // TODO: share bundles even across different bundle groups by looking if the child + // // asset is already a bundle root. In order for this to work, bundleRoots must be + // // keyed by asset + target, not just asset, so that bundles are not shared between targets. + // bundleId = + // bundleGroup.type == childAsset.type + // ? bundleGroupNodeId + // : bundleGraph + // .getNodeIdsConnectedFrom(bundleGroupNodeId) + // .find(id => { + // let node = bundleGraph.getNode(id); + // return node !== 'root' && node?.type == childAsset.type; + // }); + // } else { + entryAsset = childAsset; + //} + let referencingBundleId = nullthrows( + bundleRoots.get(referencingBundleRoot), + )[0]; + let referencingBundle = nullthrows( + bundleGraph.getNode(referencingBundleId), + ); + invariant(referencingBundle !== 'root'); let bundle; + bundleId = bundles.get(childAsset.id); if (bundleId == null) { // Create a new bundle if none of the same type exists already. bundle = createBundle({ @@ -479,16 +499,25 @@ function createIdealGraph( type: childAsset.type, env: childAsset.env, bundleBehavior: childAsset.bundleBehavior, - target: bundleGroup.target, + target: referencingBundle.target, needsStableName: childAsset.bundleBehavior === 'inline' || dependency.bundleBehavior === 'inline' || (dependency.priority === 'parallel' && !dependency.needsStableName) ? false - : bundleGroup.needsStableName, + : referencingBundle.needsStableName, }); bundleId = bundleGraph.addNode(bundle); + typeChangeIds.add(bundleId); + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } } else { // Otherwise, merge this asset into the existing bundle. bundle = bundleGraph.getNode(bundleId); @@ -502,7 +531,7 @@ function createIdealGraph( bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); + bundleGraph.addEdge(referencingBundleId, bundleId); if (bundleId != bundleGroupNodeId) { dependencyBundleGraph.addEdge( @@ -525,7 +554,7 @@ function createIdealGraph( // Add an edge from the bundle group entry to the new bundle. // This indicates that the bundle is loaded together with the entry - bundleGraph.addEdge(bundleGroupNodeId, bundleId); + //bundleGraph.addEdge(bundleGroupNodeId, bundleId); } assetReference.get(childAsset).push([dependency, bundle]); @@ -551,6 +580,55 @@ function createIdealGraph( }, }); + // Clean up type change bundles within the same bundlegroups + for (let [nodeIdA, a] of bundleGraph.nodes) { + //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes + for (let [nodeIdB, b] of bundleGraph.nodes) { + if ( + a !== 'root' && + b !== 'root' && + a !== b && + typeChangeIds.has(nodeIdA) && + typeChangeIds.has(nodeIdB) && + a.bundleBehavior !== 'inline' && + b.bundleBehavior !== 'inline' && + a.type === b.type + ) { + let bundleBbundleGroups = new Set( + bundleGraph + .getNodeIdsConnectedTo(nodeIdB) + .filter(id => bundleGroupBundleIds.has(id)), + ); + let bundleABundleGroups = new Set( + bundleGraph + .getNodeIdsConnectedTo(nodeIdA) + .filter(id => bundleGroupBundleIds.has(id)), + ); //the two css bundle don't share bundlegroups ALLL bundlegroups + //because type change bundles are considered bundlegroup starts or at least added edges + + if (setEqual(bundleBbundleGroups, bundleABundleGroups)) { + let shouldMerge = true; + for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(nodeIdB)), + ALL_EDGE_TYPES, + )) { + let depNode = dependencyBundleGraph.getNode(depId); + if ( + depNode && + depNode.type === 'dependency' && + depNode.value.specifierType === 'url' + ) { + shouldMerge = false; + continue; + } + } + if (!shouldMerge) continue; + mergeBundle(nodeIdA, nodeIdB); + } + } + } + } + // Step 3: Determine reachability for every asset from each bundleRoot. // This is later used to determine which bundles to place each asset in. for (let [root] of bundleRoots) { @@ -559,8 +637,6 @@ function createIdealGraph( } } - // Models bundleRoots and the assets that require it synchronously - let reachableRoots: ContentGraph = new ContentGraph(); for (let [root] of bundleRoots) { let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); assetGraph.traverse((node, _, actions) => { @@ -941,13 +1017,58 @@ function createIdealGraph( } } } + function deleteBundle(bundleRoot: BundleRoot) { bundleGraph.removeNode(nullthrows(bundles.get(bundleRoot.id))); bundleRoots.delete(bundleRoot); - reachableRoots.replaceNodeIdsConnectedTo( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - [], - ); + bundles.delete(bundleRoot.id); + if (reachableRoots.hasContentKey(bundleRoot.id)) { + reachableRoots.replaceNodeIdsConnectedTo( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + [], + ); + } + if (asyncBundleRootGraph.hasContentKey(bundleRoot.id)) { + asyncBundleRootGraph.removeNode( + asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + ); + } + } + + function mergeBundle(mainNodeId: NodeId, otherNodeId: NodeId) { + //merges assets of "otherRoot" into "mainBundleRoot" + let a = nullthrows(bundleGraph.getNode(mainNodeId)); + let b = nullthrows(bundleGraph.getNode(otherNodeId)); + invariant(a !== 'root' && b !== 'root'); + let bundleRootB = nullthrows(b.mainEntryAsset); + let mainBundleRoot = nullthrows(a.mainEntryAsset); + for (let asset of b.assets) { + a.assets.add(asset); + } + for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(otherNodeId)), + ALL_EDGE_TYPES, + )) { + dependencyBundleGraph.replaceNodeIdsConnectedTo(depId, [ + dependencyBundleGraph.getNodeIdByContentKey(String(mainNodeId)), + ]); + } + + //clean up asset reference + for (let dependencyTuple of assetReference.get(bundleRootB)) { + dependencyTuple[1] = a; + } + //add in any lost edges + for (let nodeId of bundleGraph.getNodeIdsConnectedTo(otherNodeId)) { + bundleGraph.addEdge(nodeId, mainNodeId); + } + deleteBundle(bundleRootB); + + bundleRoots.set(bundleRootB, [ + mainNodeId, + bundleRoots.get(mainBundleRoot)[1], + ]); + bundles.set(bundleRootB.id, mainNodeId); } function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { let bundle = bundleGraph.getNode( @@ -1055,8 +1176,9 @@ async function loadBundlerConfig( options: PluginOptions, ): Promise { let conf = await config.getConfig([], { - packageKey: '@parcel/bundler-experimental', + packageKey: '@parcel/bundler-default', }); + if (!conf) { return HTTP_OPTIONS['2']; } @@ -1069,10 +1191,10 @@ async function loadBundlerConfig( data: conf?.contents, source: await options.inputFS.readFile(conf.filePath, 'utf8'), filePath: conf.filePath, - prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-experimental')}`, + prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-default')}`, }, - '@parcel/bundler-experimental', - 'Invalid config for @parcel/bundler-experimental', + '@parcel/bundler-default', + 'Invalid config for @parcel/bundler-default', ); let http = conf.contents.http ?? 2; diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 215e64b0d6f..07548f04c61 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -1308,9 +1308,11 @@ export default class BundleGraph { // Shared bundles seem to depend on being used in the opposite order // they were added. // TODO: Should this be the case? - this._graph - .getNodeIdsConnectedFrom(nodeId, bundleGraphEdgeTypes.references) - .reverse(), + this._graph.getNodeIdsConnectedFrom( + nodeId, + bundleGraphEdgeTypes.references, + ), + //.reverse(), }); return [...referencedBundles]; diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index b8b20d51191..f09becb8774 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -2228,12 +2228,9 @@ describe('html', function () { // reuse the b.css bundle from b.html. let html = await outputFS.readFile(path.join(distDir, 'a.html'), 'utf8'); assert.equal( - html.match(//g).length, - 1, - ); - assert.equal( - html.match(//g).length, - 1, + html.match(//g) + .length, + 2, ); // a.html should reference a.js only @@ -2242,7 +2239,7 @@ describe('html', function () { assert.equal(html.match(/b\.[a-z0-9]+\.js/g), null); let css = await outputFS.readFile( - path.join(distDir, html.match(/\/a\.[a-z0-9]+\.css/)[0]), + path.join(distDir, html.match(/\/\w+\.[a-z0-9]+\.css/g)[0]), 'utf8', ); assert(css.includes('.a {')); @@ -2252,11 +2249,8 @@ describe('html', function () { // It should not point to the bundle containing a.css from a.html html = await outputFS.readFile(path.join(distDir, 'b.html'), 'utf8'); assert.equal( - html.match(//g), - null, - ); - assert.equal( - html.match(//g).length, + html.match(//g) + .length, 1, ); @@ -2266,7 +2260,7 @@ describe('html', function () { assert.equal(html.match(/b\.[a-z0-9]+\.js/g).length, 1); css = await outputFS.readFile( - path.join(distDir, html.match(/\/b\.[a-z0-9]+\.css/)[0]), + path.join(distDir, html.match(/\/\w+\.[a-z0-9]+\.css/)[0]), 'utf8', ); assert(!css.includes('.a {')); diff --git a/packages/core/utils/src/collection.js b/packages/core/utils/src/collection.js index 8c62599f869..7d9e5c8556e 100644 --- a/packages/core/utils/src/collection.js +++ b/packages/core/utils/src/collection.js @@ -55,3 +55,15 @@ export function setIntersect(a: Set, b: Set): void { export function setUnion(a: Iterable, b: Iterable): Set { return new Set([...a, ...b]); } + +export function setEqual(a: Set, b: Set): boolean { + if (a.size != b.size) { + return false; + } + for (let entry of a) { + if (!b.has(entry)) { + return false; + } + } + return true; +} diff --git a/packages/core/utils/src/index.js b/packages/core/utils/src/index.js index 4df6ac1adf0..78c98e124fd 100644 --- a/packages/core/utils/src/index.js +++ b/packages/core/utils/src/index.js @@ -35,6 +35,7 @@ export { objectSortedEntries, objectSortedEntriesDeep, setDifference, + setEqual, setIntersect, setUnion, } from './collection'; From 183e9e7577f829b327f06b7321f3c1e1310ec77f Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 11 May 2022 16:38:09 -0400 Subject: [PATCH 62/87] extract shared bundles from inline bundles --- packages/bundlers/experimental/src/ExperimentalBundler.js | 2 -- 1 file changed, 2 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 0cee3388f23..723bff3dfb1 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -784,7 +784,6 @@ function createIdealGraph( entries.has(a) || !a.isBundleSplittable || getBundleFromBundleRoot(a).needsStableName || - getBundleFromBundleRoot(a).bundleBehavior === 'inline' || getBundleFromBundleRoot(a).bundleBehavior === 'isolated', ); reachable = reachable.filter( @@ -792,7 +791,6 @@ function createIdealGraph( !entries.has(a) && a.isBundleSplittable && !getBundleFromBundleRoot(a).needsStableName && - getBundleFromBundleRoot(a).bundleBehavior !== 'inline' && getBundleFromBundleRoot(a).bundleBehavior !== 'isolated', ); From 7ee8435f825bbf73c1165e42c61a1986599221cd Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Wed, 11 May 2022 21:14:46 -0400 Subject: [PATCH 63/87] Internalize all async bundle before placing assets into bundles, add internalized assets to share bundles from their source bundles --- .../experimental/src/ExperimentalBundler.js | 124 +++++++----------- 1 file changed, 50 insertions(+), 74 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 723bff3dfb1..ba02fdf2d67 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -769,11 +769,47 @@ function createIdealGraph( } } + // Step Internalize async bundles + for (let [id, bundleRoot] of asyncBundleRootGraph.nodes) { + if (bundleRoot === 'root') continue; + let parentRoots = asyncBundleRootGraph + .getNodeIdsConnectedTo(id) + .map(id => nullthrows(asyncBundleRootGraph.getNode(id))); + let canDelete = + getBundleFromBundleRoot(bundleRoot).bundleBehavior !== 'isolated'; + if (parentRoots.length === 0) continue; + for (let parent of parentRoots) { + if (parent === 'root') { + canDelete = false; + continue; + } + if ( + reachableRoots.hasEdge( + reachableRoots.getNodeIdByContentKey(parent.id), + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + ) || + asyncAncestorAssets.get(parent)?.has(bundleRoot) + ) { + let parentBundle = bundleGraph.getNode( + nullthrows(bundles.get(parent.id)), + ); + invariant(parentBundle != null && parentBundle !== 'root'); + parentBundle.internalizedAssetIds.push(bundleRoot.id); + } else { + canDelete = false; + } + } + if (canDelete) { + deleteBundle(bundleRoot); + } + } + // Step 5: Place all assets into bundles or create shared bundles. Each asset // is placed into a single bundle based on the bundle entries it is reachable from. // This creates a maximally code split bundle graph with no duplication. for (let asset of assets) { // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means + let reachable: Array = getReachableBundleRoots( asset, reachableRoots, @@ -831,80 +867,6 @@ function createIdealGraph( let entryBundleId = nullthrows(bundles.get(entry.id)); let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); invariant(entryBundle !== 'root'); - - // If this asset is associated with a bundle, we must attempt to clean up - let internalizedBundleId = bundles.get(asset.id); - if (internalizedBundleId !== undefined) { - let toBeInternalizedNodeId = asyncBundleRootGraph.getNodeIdByContentKey( - asset.id, - ); - let bundleRootsRequiring = nullthrows( - asyncBundleRootGraph.getNodeIdsConnectedTo(toBeInternalizedNodeId), - ); - //attempt to clean up for all immediate bundle connections - for (let parentBundleId of bundleRootsRequiring) { - let parentBundleRoot = asyncBundleRootGraph.getNode(parentBundleId); - if (parentBundleRoot === 'root') continue; - invariant(parentBundleRoot != null); - // we may remove the bundle's connection to anything that already has it from previous ancestors, i.e. asyncAnc - if (asyncAncestorAssets.get(parentBundleRoot)?.has(asset)) { - let bundleId = nullthrows(bundles.get(parentBundleRoot.id)); - let b = bundleGraph.getNode(bundleId); - if (b && b !== 'root') { - b.internalizedAssetIds.push(asset.id); - } - - asyncBundleRootGraph.removeEdge( - parentBundleId, - toBeInternalizedNodeId, - ); - let remainingConnections = asyncBundleRootGraph.getNode( - toBeInternalizedNodeId, - ) - ? asyncBundleRootGraph - .getNodeIdsConnectedTo(toBeInternalizedNodeId) - .map(id => asyncBundleRootGraph.getNode(id)) - : []; - let containsRoot = remainingConnections.find( - node => node === 'root', - ); - if (remainingConnections.length === 1 && containsRoot) { - asyncBundleRootGraph.removeNode(toBeInternalizedNodeId); - } - if ( - !asyncBundleRootGraph.hasNode(toBeInternalizedNodeId) && - nullthrows(bundleGraph.getNode(toBeInternalizedNodeId)) - .bundleBehavior !== 'isolated' - ) { - deleteBundle(asset); - } - } else if (parentBundleRoot === entry) { - //entries do not have asyncAncestors so we must check if its available sync - let hasSync = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(entry.id), - ) - .some(id => { - let a = reachableRoots.getNode(id); - return a === asset; - }); - if (hasSync) { - entryBundle.internalizedAssetIds.push(asset.id); - asyncBundleRootGraph.removeEdge( - asyncBundleRootGraph.getNodeIdByContentKey(entry.id), - toBeInternalizedNodeId, - ); - } - if ( - !asyncBundleRootGraph.hasNode(toBeInternalizedNodeId) && - nullthrows(bundleGraph.getNode(toBeInternalizedNodeId)) - .bundleBehavior !== 'isolated' - ) { - deleteBundle(asset); - } - } - } - } entryBundle.assets.add(asset); entryBundle.size += asset.stats.size; } @@ -926,6 +888,20 @@ function createIdealGraph( env: firstSourceBundle.env, }); bundle.sourceBundles = sourceBundles; + let sharedInternalizedAssets = new Set( + firstSourceBundle.internalizedAssetIds, + ); + + for (let p of sourceBundles) { + let parentBundle = nullthrows(bundleGraph.getNode(p)); + invariant(parentBundle !== 'root'); + if (parentBundle === firstSourceBundle) continue; + setIntersect( + sharedInternalizedAssets, + new Set(parentBundle.internalizedAssetIds), + ); + } + bundle.internalizedAssetIds = [...sharedInternalizedAssets]; bundleId = bundleGraph.addNode(bundle); bundles.set(key, bundleId); } else { From ec2ec0a9c0d237bd2940b0aef0dd02d2310c2b79 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Fri, 13 May 2022 11:11:16 -0400 Subject: [PATCH 64/87] add assert bundles to test --- packages/core/integration-tests/test/html.js | 23 +++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/packages/core/integration-tests/test/html.js b/packages/core/integration-tests/test/html.js index f09becb8774..135b1bdbf60 100644 --- a/packages/core/integration-tests/test/html.js +++ b/packages/core/integration-tests/test/html.js @@ -2188,7 +2188,28 @@ describe('html', function () { }, ); let bundles = b.getBundles(); - + assertBundles(b, [ + { + name: 'index.html', + type: 'html', + assets: ['index.html'], + }, + { + type: 'js', + assets: [ + 'index.js', + 'index.js', + 'index.js', + 'index.js', + 'client.js', + 'bundle-manifest.js', + ], + }, + { + type: 'js', + assets: ['viewer.js'], + }, + ]); let html = await outputFS.readFile( path.join(distDir, 'index.html'), 'utf8', From 335e31edf6ccba3d54ae96355b27ab3feb227c4b Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Fri, 13 May 2022 18:12:50 -0400 Subject: [PATCH 65/87] Reverse merge asset insertion order to maintain dep order for css, remove addassetonexit, instead do upward dfs to determine bundlegroups for assets --- .../experimental/src/ExperimentalBundler.js | 80 ++++++------------- 1 file changed, 23 insertions(+), 57 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index ba02fdf2d67..f6e5fe27597 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -346,10 +346,6 @@ function createIdealGraph( } let assets = []; - let assetsToAddOnExit: DefaultMap< - Dependency, - Array<[Bundle, Asset]>, - > = new DefaultMap(() => []); let typeChangeIds = new Set(); // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, @@ -458,28 +454,6 @@ function createIdealGraph( // Find an existing bundle of the same type within the bundle group. let bundleId; - let entryAsset; - let uniqueKey; - // if ( - // childAsset.bundleBehavior !== 'inline' && - // dependency.priority !== 'parallel' - // ) { - // uniqueKey = childAsset.id; - // // TODO: share bundles even across different bundle groups by looking if the child - // // asset is already a bundle root. In order for this to work, bundleRoots must be - // // keyed by asset + target, not just asset, so that bundles are not shared between targets. - // bundleId = - // bundleGroup.type == childAsset.type - // ? bundleGroupNodeId - // : bundleGraph - // .getNodeIdsConnectedFrom(bundleGroupNodeId) - // .find(id => { - // let node = bundleGraph.getNode(id); - // return node !== 'root' && node?.type == childAsset.type; - // }); - // } else { - entryAsset = childAsset; - //} let referencingBundleId = nullthrows( bundleRoots.get(referencingBundleRoot), )[0]; @@ -490,12 +464,10 @@ function createIdealGraph( let bundle; bundleId = bundles.get(childAsset.id); if (bundleId == null) { - // Create a new bundle if none of the same type exists already. bundle = createBundle({ // We either have an entry asset or a unique key. // Bundles created from type changes shouldn't have an entry asset. - asset: entryAsset, - uniqueKey, + asset: childAsset, type: childAsset.type, env: childAsset.env, bundleBehavior: childAsset.bundleBehavior, @@ -524,12 +496,10 @@ function createIdealGraph( invariant(bundle != null && bundle !== 'root'); } - if (!entryAsset) { - // Queue the asset to be added on exit of this node, so we add dependencies first. - assetsToAddOnExit.get(dependency).push([bundle, childAsset]); - } - bundles.set(childAsset.id, bundleId); + // This may be wrong + // A bundle can belong to multiple bundlegroups, all teh bundle groups of it's + // ancestors, and all async and entry bundles before it are " bundle groups " bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); bundleGraph.addEdge(referencingBundleId, bundleId); @@ -565,15 +535,6 @@ function createIdealGraph( return node; }, exit(node) { - if (node.type === 'dependency' && assetsToAddOnExit.has(node.value)) { - let assetsToAdd = assetsToAddOnExit.get(node.value); - for (let [bundle, asset] of assetsToAdd) { - bundle.assets.add(asset); - bundle.size += asset.stats.size; - } - assetsToAddOnExit.delete(node.value); - } - if (stack[stack.length - 1]?.[0] === node.value) { stack.pop(); } @@ -594,18 +555,9 @@ function createIdealGraph( b.bundleBehavior !== 'inline' && a.type === b.type ) { - let bundleBbundleGroups = new Set( - bundleGraph - .getNodeIdsConnectedTo(nodeIdB) - .filter(id => bundleGroupBundleIds.has(id)), - ); - let bundleABundleGroups = new Set( - bundleGraph - .getNodeIdsConnectedTo(nodeIdA) - .filter(id => bundleGroupBundleIds.has(id)), - ); //the two css bundle don't share bundlegroups ALLL bundlegroups - //because type change bundles are considered bundlegroup starts or at least added edges - + //should be the set of all + let bundleBbundleGroups = getBundleGroupsForBundle(nodeIdB); + let bundleABundleGroups = getBundleGroupsForBundle(nodeIdA); if (setEqual(bundleBbundleGroups, bundleABundleGroups)) { let shouldMerge = true; for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( @@ -1008,6 +960,19 @@ function createIdealGraph( ); } } + function getBundleGroupsForBundle(nodeId: NodeId) { + let bundleGroupBundleIds = new Set(); + bundleGraph.traverseAncestors(nodeId, ancestorId => { + if ( + bundleGraph + .getNodeIdsConnectedTo(ancestorId) + .includes(bundleGraph.rootNodeId) + ) { + bundleGroupBundleIds.add(ancestorId); + } + }); + return bundleGroupBundleIds; + } function mergeBundle(mainNodeId: NodeId, otherNodeId: NodeId) { //merges assets of "otherRoot" into "mainBundleRoot" @@ -1016,9 +981,10 @@ function createIdealGraph( invariant(a !== 'root' && b !== 'root'); let bundleRootB = nullthrows(b.mainEntryAsset); let mainBundleRoot = nullthrows(a.mainEntryAsset); - for (let asset of b.assets) { - a.assets.add(asset); + for (let asset of a.assets) { + b.assets.add(asset); } + a.assets = b.assets; for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( dependencyBundleGraph.getNodeIdByContentKey(String(otherNodeId)), ALL_EDGE_TYPES, From f7a2296c1125fe29a19a7e919c8c609751edc1c5 Mon Sep 17 00:00:00 2001 From: Agnieszka Gawrys Date: Mon, 23 May 2022 15:17:54 -0400 Subject: [PATCH 66/87] Forked tests which rely on size calculation for shared bundles which is different across bundles and rename bundles steps without numbers --- .../experimental/src/ExperimentalBundler.js | 19 +- .../integration-tests/test/css-modules.js | 105 ++++++++---- packages/core/integration-tests/test/html.js | 162 ++++++++++++------ .../integration-tests/test/scope-hoisting.js | 104 +++++++---- 4 files changed, 262 insertions(+), 128 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index f6e5fe27597..c00b4ffc683 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -295,7 +295,7 @@ function createIdealGraph( // Models bundleRoots and the assets that require it synchronously let reachableRoots: ContentGraph = new ContentGraph(); - // Step 1: Find and create bundles for entries from assetGraph + // Step Create Entry Bundles: Find and create bundles for entries from assetGraph let entries: Map = new Map(); let sharedToSourceBundleIds: Map> = new Map(); @@ -348,7 +348,7 @@ function createIdealGraph( let assets = []; let typeChangeIds = new Set(); - // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, + // Step Create Bundles: Traverse the asset graph and create bundles for asset type changes and async dependencies, // only adding the entry asset of each bundle, not the subgraph. assetGraph.traverse({ enter(node, context, actions) { @@ -541,7 +541,7 @@ function createIdealGraph( }, }); - // Clean up type change bundles within the same bundlegroups + // Step Merge Type Change Bundles: Clean up type change bundles within the same bundlegroups for (let [nodeIdA, a] of bundleGraph.nodes) { //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes for (let [nodeIdB, b] of bundleGraph.nodes) { @@ -555,7 +555,6 @@ function createIdealGraph( b.bundleBehavior !== 'inline' && a.type === b.type ) { - //should be the set of all let bundleBbundleGroups = getBundleGroupsForBundle(nodeIdB); let bundleABundleGroups = getBundleGroupsForBundle(nodeIdA); if (setEqual(bundleBbundleGroups, bundleABundleGroups)) { @@ -581,7 +580,7 @@ function createIdealGraph( } } - // Step 3: Determine reachability for every asset from each bundleRoot. + // Step Determine Reachability: Determine reachability for every asset from each bundleRoot. // This is later used to determine which bundles to place each asset in. for (let [root] of bundleRoots) { if (!entries.has(root)) { @@ -650,12 +649,12 @@ function createIdealGraph( // and the bundleRoots reachable from each of these assets let asyncAncestorAssets: Map> = new Map(); - // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group for (let entry of entries.keys()) { // Initialize an empty set of ancestors available to entries asyncAncestorAssets.set(entry, new Set()); } + // Step Determine Availability // Visit nodes in a topological order, visiting parent nodes before child nodes. // This allows us to construct an understanding of which assets will already be // loaded and available when a bundle runs, by pushing available assets downwards and @@ -756,7 +755,7 @@ function createIdealGraph( } } - // Step 5: Place all assets into bundles or create shared bundles. Each asset + // Step Insert Or Share: Place all assets into bundles or create shared bundles. Each asset // is placed into a single bundle based on the bundle entries it is reachable from. // This creates a maximally code split bundle graph with no duplication. for (let asset of assets) { @@ -877,7 +876,7 @@ function createIdealGraph( } } - // Step 7: Merge any shared bundles under the minimum bundle size back into + // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into // their source bundles, and remove the bundle. for (let [bundleNodeId, bundle] of bundleGraph.nodes) { if (bundle === 'root') continue; @@ -887,7 +886,7 @@ function createIdealGraph( } } - // Step 8: Remove shared bundles from bundle groups that hit the parallel request limit. + // Step Parallel Request Limit Cleanup: Remove shared bundles from bundle groups that hit the parallel request limit. for (let [bundleId, bundleGroupId] of bundleRoots.values()) { // Only handle bundle group entries. if (bundleId != bundleGroupId) { @@ -895,7 +894,7 @@ function createIdealGraph( } // Find the bundles in this bundle group. - let bundleIdsInGroup = bundleGraph.getNodeIdsConnectedFrom(bundleGroupId); + let bundleIdsInGroup = [...getBundleGroupsForBundle(bundleGroupId)]; if (bundleIdsInGroup.length > config.maxParallelRequests) { // Sort the bundles so the smallest ones are removed first. let bundlesInGroup = bundleIdsInGroup diff --git a/packages/core/integration-tests/test/css-modules.js b/packages/core/integration-tests/test/css-modules.js index 26ace5c1642..8108e725133 100644 --- a/packages/core/integration-tests/test/css-modules.js +++ b/packages/core/integration-tests/test/css-modules.js @@ -545,7 +545,7 @@ describe('css modules', () => { }, ]); }); - + // Forked because experimental bundler will not merge bundles of same types if they do not share all their bundlegroups it('should handle @import in css modules', async function () { let b = await bundle( [ @@ -577,38 +577,77 @@ describe('css modules', () => { assert.deepEqual(res, [['page2', '_4fY2uG_foo _1ZEqVW_foo j1UkRG_foo']]); - assertBundles(b, [ - { - name: 'page1.html', - assets: ['page1.html'], - }, - { - name: 'page2.html', - assets: ['page2.html'], - }, - { - type: 'js', - assets: [ - 'page1.js', - 'index.module.css', - 'a.module.css', - 'b.module.css', - ], - }, - { - type: 'js', - assets: [ - 'page2.js', - 'index.module.css', - 'a.module.css', - 'b.module.css', - ], - }, - { - type: 'css', - assets: ['index.module.css', 'a.module.css', 'b.module.css'], - }, - ]); + if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { + assertBundles(b, [ + { + name: 'page1.html', + assets: ['page1.html'], + }, + { + name: 'page2.html', + assets: ['page2.html'], + }, + { + type: 'js', + assets: [ + 'page1.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'js', + assets: [ + 'page2.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'css', + assets: ['a.module.css', 'b.module.css'], + }, + { + type: 'css', + assets: ['index.module.css'], + }, + ]); + } else { + assertBundles(b, [ + { + name: 'page1.html', + assets: ['page1.html'], + }, + { + name: 'page2.html', + assets: ['page2.html'], + }, + { + type: 'js', + assets: [ + 'page1.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'js', + assets: [ + 'page2.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'css', + assets: ['index.module.css', 'a.module.css', 'b.module.css'], + }, + ]); + } }); it('should not process inline