From 1c30b0a2c7c686e323d40e68d9bc1b5ec5b3d80b Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 23 Mar 2022 18:58:11 -0700 Subject: [PATCH 01/12] implement parallel request limits --- .../experimental/src/ExperimentalBundler.js | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 99719dba6c6..1a17e0e8672 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -789,6 +789,24 @@ function createIdealGraph( let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); let bundle; + + sourceBundles.filter(sourceBundleId => { + if (bundleId !== sourceBundleId) { + let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); + invariant(sourceBundle !== 'root'); + let bundleGroupIds = bundleGraph + .getNodeIdsConnectedTo(sourceBundleId) + .filter(n => bundleGraph.getNode(n) !== 'root'); + // Check that all bundle groups the source bundle belongs to + // are within the parallel request limit + return bundleGroupIds.every( + groupId => + bundleGraph.getNodeIdsConnectedFrom(groupId).length < + config.maxParallelRequests, + ); + } + }); + if (bundleId == null) { let firstSourceBundle = nullthrows( bundleGraph.getNode(sourceBundles[0]), @@ -808,7 +826,6 @@ function createIdealGraph( } bundle.assets.add(asset); bundle.size += asset.stats.size; - for (let sourceBundleId of sourceBundles) { if (bundleId !== sourceBundleId) { bundleGraph.addEdge(sourceBundleId, bundleId); From f8a8e65b9d2408c1ad8f443b550c13991e5da684 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Thu, 24 Mar 2022 16:42:38 -0700 Subject: [PATCH 02/12] cleanup --- packages/bundlers/experimental/src/ExperimentalBundler.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 1a17e0e8672..5c7279c5a4a 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -788,7 +788,6 @@ function createIdealGraph( let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); - let bundle; sourceBundles.filter(sourceBundleId => { if (bundleId !== sourceBundleId) { @@ -807,6 +806,7 @@ function createIdealGraph( } }); + let bundle; if (bundleId == null) { let firstSourceBundle = nullthrows( bundleGraph.getNode(sourceBundles[0]), @@ -826,6 +826,7 @@ function createIdealGraph( } bundle.assets.add(asset); bundle.size += asset.stats.size; + for (let sourceBundleId of sourceBundles) { if (bundleId !== sourceBundleId) { bundleGraph.addEdge(sourceBundleId, bundleId); From 892edd37107a8032ae33466b721570efec3e556f Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Mon, 28 Mar 2022 18:09:24 -0700 Subject: [PATCH 03/12] create all shared bundles first then remove later --- .../experimental/src/ExperimentalBundler.js | 75 ++++++++++++++----- 1 file changed, 57 insertions(+), 18 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 5c7279c5a4a..421722a1a73 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -788,24 +788,6 @@ function createIdealGraph( let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); let key = reachable.map(a => a.id).join(','); let bundleId = bundles.get(key); - - sourceBundles.filter(sourceBundleId => { - if (bundleId !== sourceBundleId) { - let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); - invariant(sourceBundle !== 'root'); - let bundleGroupIds = bundleGraph - .getNodeIdsConnectedTo(sourceBundleId) - .filter(n => bundleGraph.getNode(n) !== 'root'); - // Check that all bundle groups the source bundle belongs to - // are within the parallel request limit - return bundleGroupIds.every( - groupId => - bundleGraph.getNodeIdsConnectedFrom(groupId).length < - config.maxParallelRequests, - ); - } - }); - let bundle; if (bundleId == null) { let firstSourceBundle = nullthrows( @@ -852,6 +834,63 @@ function createIdealGraph( } } + // Step 8: Remove shared bundles from bundle groups that hit the parallel request limit. + for (let [bundleId, bundleGroupId] of bundleRoots.values()) { + // Only handle bundle group entries. + if (bundleId != bundleGroupId) { + continue; + } + + // Find the bundles in this bundle group. + let bundleIdsInGroup = bundleGraph.getNodeIdsConnectedFrom(bundleGroupId); + if (bundleIdsInGroup.length > config.maxParallelRequests) { + // Sort the bundles so the smallest ones are removed first. + let bundlesInGroup = bundleIdsInGroup + .map(id => nullthrows(bundleGraph.getNode(id))) + .map(bundle => { + // For Flow + invariant(bundle !== 'root'); + return bundle; + }) + .sort((a, b) => a.size - b.size); + + // Remove bundles until the bundle group is within the parallel request limit. + for ( + let i = 0; + i < bundlesInGroup.length - config.maxParallelRequests; + i++ + ) { + let bundleId = bundleIdsInGroup[i]; + let bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); + // Add all assets in the shared bundle into the source bundles that are within this bundle group. + let sourceBundles = bundle.sourceBundles + .filter(b => bundlesInGroup.includes(b)) + .map(id => nullthrows(bundleGraph.getNode(id))); + + for (let sourceBundle of sourceBundles) { + invariant(sourceBundle !== 'root'); + for (let asset of bundle.assets) { + sourceBundle.assets.add(asset); + sourceBundle.size += asset.stats.size; + } + } + + // Remove the edge from this bundle group to the shared bundle. + bundleGraph.removeEdge(bundleGroupId, bundleId); + // If there is now only a single bundle group that contains this bundle, + // merge it into the remaining source bundles. If it is orphaned entirely, remove it. + let incomingNodeCount = + bundleGraph.getNodeIdsConnectedTo(bundleId).length; + if (incomingNodeCount === 1) { + removeBundle(bundleGraph, bundleId); + } else if (incomingNodeCount == 0) { + bundleGraph.removeNode(bundleId); + } + } + } + } + return { bundleGraph, dependencyBundleGraph, From 7faa3d738f17d1a710963fd0e6f47cd9d0959a77 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Mon, 2 May 2022 11:08:41 -0700 Subject: [PATCH 04/12] remove asset references for deleted bundles + minor fixes --- .../experimental/src/ExperimentalBundler.js | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index cf45f4220fc..7b127ef846c 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -878,7 +878,7 @@ function createIdealGraph( if (bundle === 'root') continue; if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { sharedToSourceBundleIds.delete(bundleNodeId); - removeBundle(bundleGraph, bundleNodeId); + removeBundle(bundleGraph, bundleNodeId, assetReference); } } @@ -931,7 +931,7 @@ function createIdealGraph( let incomingNodeCount = bundleGraph.getNodeIdsConnectedTo(bundleId).length; if (incomingNodeCount === 1) { - removeBundle(bundleGraph, bundleId); + removeBundle(bundleGraph, bundleId, assetReference); } else if (incomingNodeCount === 0) { bundleGraph.removeNode(bundleId); } @@ -1007,11 +1007,18 @@ function createBundle(opts: {| }; } -function removeBundle(bundleGraph: Graph, bundleId: NodeId) { +function removeBundle( + bundleGraph: Graph, + bundleId: NodeId, + assetReference: DefaultMap>, +) { let bundle = nullthrows(bundleGraph.getNode(bundleId)); invariant(bundle !== 'root'); - for (let asset of bundle.assets) { + assetReference.set( + asset, + assetReference.get(asset).filter(t => !t.includes(bundle)), + ); for (let sourceBundleId of bundle.sourceBundles) { let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); invariant(sourceBundle !== 'root'); @@ -1028,7 +1035,7 @@ async function loadBundlerConfig( options: PluginOptions, ): Promise { let conf = await config.getConfig([], { - packageKey: '@parcel/bundler-default', + packageKey: '@parcel/bundler-experimental', }); if (!conf) { return HTTP_OPTIONS['2']; @@ -1042,10 +1049,10 @@ async function loadBundlerConfig( data: conf?.contents, source: await options.inputFS.readFile(conf.filePath, 'utf8'), filePath: conf.filePath, - prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-default')}`, + prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-experimental')}`, }, - '@parcel/bundler-default', - 'Invalid config for @parcel/bundler-default', + '@parcel/bundler-experimental', + 'Invalid config for @parcel/bundler-experimental', ); let http = conf.contents.http ?? 2; From b8060e99d92eb389ae2cd6a980520f566ae167b2 Mon Sep 17 00:00:00 2001 From: Gora Kong Date: Wed, 1 Jun 2022 18:05:27 -0700 Subject: [PATCH 05/12] revise method of getting shared bundles from bundlegroup + integration test --- .../experimental/src/ExperimentalBundler.js | 24 +++++++++-- packages/core/integration-tests/test/cache.js | 40 +++++++++++++++++++ .../test/integration/large-bundlegroup/a.js | 1 + .../test/integration/large-bundlegroup/b.js | 1 + .../test/integration/large-bundlegroup/c.js | 1 + .../test/integration/large-bundlegroup/d.js | 1 + .../integration/large-bundlegroup/index.js | 4 ++ .../large-bundlegroup/package.json | 1 + .../integration/large-bundlegroup/yarn.lock | 0 9 files changed, 69 insertions(+), 4 deletions(-) create mode 100644 packages/core/integration-tests/test/integration/large-bundlegroup/a.js create mode 100644 packages/core/integration-tests/test/integration/large-bundlegroup/b.js create mode 100644 packages/core/integration-tests/test/integration/large-bundlegroup/c.js create mode 100644 packages/core/integration-tests/test/integration/large-bundlegroup/d.js create mode 100644 packages/core/integration-tests/test/integration/large-bundlegroup/index.js create mode 100644 packages/core/integration-tests/test/integration/large-bundlegroup/package.json create mode 100644 packages/core/integration-tests/test/integration/large-bundlegroup/yarn.lock diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 264f6b607b0..0b167e4add7 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -778,15 +778,24 @@ function createIdealGraph( } } - // Step 8: Remove shared bundles from bundle groups that hit the parallel request limit. + // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit. for (let [bundleId, bundleGroupId] of bundleRoots.values()) { // Only handle bundle group entries. - if (bundleId != bundleGroupId) { + if (bundleId !== bundleGroupId) { continue; } - // Find the bundles in this bundle group. - let bundleIdsInGroup = bundleGraph.getNodeIdsConnectedFrom(bundleGroupId); + // Find shared bundles in this bundle group. + let bundleIdsInGroup = []; + for (let [ + sharedBundleId, + sourceBundleIds, + ] of sharedToSourceBundleIds.entries()) { + if (sourceBundleIds.includes(bundleId)) { + bundleIdsInGroup.push(sharedBundleId); + } + } + if (bundleIdsInGroup.length > config.maxParallelRequests) { // Sort the bundles so the smallest ones are removed first. let bundlesInGroup = bundleIdsInGroup @@ -807,6 +816,7 @@ function createIdealGraph( let bundleId = bundleIdsInGroup[i]; let bundle = nullthrows(bundleGraph.getNode(bundleId)); invariant(bundle !== 'root'); + // Add all assets in the shared bundle into the source bundles that are within this bundle group. let sourceBundles = bundle.sourceBundles .filter(b => bundlesInGroup.includes(b)) @@ -822,12 +832,18 @@ function createIdealGraph( // Remove the edge from this bundle group to the shared bundle. bundleGraph.removeEdge(bundleGroupId, bundleId); + // If there is now only a single bundle group that contains this bundle, // merge it into the remaining source bundles. If it is orphaned entirely, remove it. let incomingNodeCount = bundleGraph.getNodeIdsConnectedTo(bundleId).length; if (incomingNodeCount === 1) { removeBundle(bundleGraph, bundleId, assetReference); + for (let sharedBundleId of sharedToSourceBundleIds.keys()) { + if (sharedBundleId === bundleId) { + sharedToSourceBundleIds.delete(sharedBundleId); + } + } } else if (incomingNodeCount === 0) { bundleGraph.removeNode(bundleId); } diff --git a/packages/core/integration-tests/test/cache.js b/packages/core/integration-tests/test/cache.js index b03c13689e8..544938972f2 100644 --- a/packages/core/integration-tests/test/cache.js +++ b/packages/core/integration-tests/test/cache.js @@ -4404,6 +4404,46 @@ describe('cache', function () { assert.equal(html.match(/