diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d3de8dcd6ff..8e0a2d70ad2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -91,12 +91,48 @@ jobs: uses: actions/upload-artifact@v2 with: name: Integration tests (${{matrix.os}}, node ${{matrix.node}}) - path: '**/junit-*.xml' + path: "**/junit-*.xml" + + experimental_bundler_integration_tests: + name: Experimental Bundler Integration tests (${{matrix.os}}, Node ${{matrix.node}}) + strategy: + matrix: + node: [14, 16] + os: [ubuntu-latest, macos-latest, windows-latest] + # These tend to be quite flakey, so one failed instance shouldn't stop + # others from potentially succeeding + fail-fast: false + runs-on: ${{matrix.os}} + steps: + - uses: actions/checkout@v2 + - uses: actions/setup-node@v2 + with: + cache: yarn + node-version: ${{matrix.node}} + - uses: actions-rs/toolchain@v1 + - uses: Swatinem/rust-cache@v1 + - name: Bump max inotify watches (Linux only) + run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p; + if: ${{matrix.os == 'ubuntu-latest'}} + - run: yarn --frozen-lockfile + - run: yarn build-native-release + - run: yarn test:integration-ci + env: + PARCEL_TEST_EXPERIMENTAL_BUNDLER: 1 + # Similar to + # https://github.com/marketplace/actions/publish-unit-test-results#use-with-matrix-strategy + - name: Upload JUnit results + if: always() + uses: actions/upload-artifact@v2 + with: + name: Exprimental Bundler Integration tests (${{matrix.os}}, node ${{matrix.node}}) + path: "**/junit-*.xml" test_report: name: Test report runs-on: ubuntu-latest - needs: [unit_tests, integration_tests] + needs: + [unit_tests, integration_tests, experimental_bundler_integration_tests] if: always() steps: - name: Create test report diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 89b9a497fab..b0c833cbec6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -27,7 +27,7 @@ yarn install yarn build-native ``` -If you want, you can create a temporary example for debugging in the folder `packages/examples`. You can start by copying the `simple` example and try to reproduce the bug. It has everything set up for working on local changes and you can run `yarn build` to build the project. If you're re-using another example or creating one from scratch, make sure to use the `--no-cache` flag for `parcel build` to see your local changes reflected. *Please don't commit this example.* +If you want, you can create a temporary example for debugging in the folder `packages/examples`. You can start by copying the `simple` example and try to reproduce the bug. It has everything set up for working on local changes and you can run `yarn build` to build the project. If you're re-using another example or creating one from scratch, make sure to use the `--no-cache` flag for `parcel build` to see your local changes reflected. _Please don't commit this example._ After you've figured out where the issue originated from and found a fix, try to add a test case or ask for help on how to proceed if the use case is more complex. diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js index 12d85953d9a..920f8f3a5e1 100644 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ b/packages/bundlers/experimental/src/ExperimentalBundler.js @@ -19,7 +19,13 @@ import {ContentGraph, Graph} from '@parcel/graph'; import invariant from 'assert'; import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; -import {setIntersect, validateSchema, DefaultMap} from '@parcel/utils'; +import { + setIntersect, + setUnion, + setEqual, + validateSchema, + DefaultMap, +} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -51,6 +57,8 @@ const HTTP_OPTIONS = { }; type AssetId = string; + +/* BundleRoot - An asset that is the main entry of a Bundle. */ type BundleRoot = Asset; export type Bundle = {| uniqueKey: ?string, @@ -83,21 +91,47 @@ type DependencyBundleGraph = ContentGraph< |}, number, >; +// IdealGraph is the structure we will pass to decorate, +// which mutates the assetGraph into the bundleGraph we would +// expect from default bundler type IdealGraph = {| dependencyBundleGraph: DependencyBundleGraph, bundleGraph: Graph, - bundleGroupBundleIds: Array, + bundleGroupBundleIds: Set, assetReference: DefaultMap>, sharedToSourceBundleIds: Map>, |}; +/** + * + * The Bundler works by creating an IdealGraph, which contains a BundleGraph that models bundles + * connected to othervbundles by what references them, and thus models BundleGroups. + * + * First, we enter `bundle({bundleGraph, config})`. Here, "bundleGraph" is actually just the + * assetGraph turned into a type `MutableBundleGraph`, which will then be mutated in decorate, + * and turned into what we expect the bundleGraph to be as per the old (default) bundler structure + * & what the rest of Parcel expects a BundleGraph to be. + * + * `bundle({bundleGraph, config})` First gets a Mapping of target to entries, In most cases there is + * only one target, and one or more entries. (Targets are pertinent in monorepos or projects where you + * will have two or more distDirs, or output folders.) Then calls create IdealGraph and Decorate per target. + * + */ export default (new Bundler({ loadConfig({config, options}) { return loadBundlerConfig(config, options); }, bundle({bundleGraph, config}) { - decorateLegacyGraph(createIdealGraph(bundleGraph, config), bundleGraph); + let targetMap = getEntryByTarget(bundleGraph); // Organize entries by target output folder/ distDir + let graphs = []; + for (let entries of targetMap.values()) { + // Create separate bundleGraphs per distDir + graphs.push(createIdealGraph(bundleGraph, config, entries)); + } + for (let g of graphs) { + decorateLegacyGraph(g, bundleGraph); //mutate original graph + } }, optimize() {}, }): Bundler); @@ -116,19 +150,17 @@ function decorateLegacyGraph( } = idealGraph; let entryBundleToBundleGroup: Map = new Map(); - // Step 1: Create bundle groups, bundles, and shared bundles and add assets to them + // Step Create Bundles: Create bundle groups, bundles, and shared bundles and add assets to them for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { if (idealBundle === 'root') continue; let entryAsset = idealBundle.mainEntryAsset; let bundleGroup; let bundle; - if (bundleGroupBundleIds.includes(bundleNodeId)) { - invariant(entryAsset != null); + if (bundleGroupBundleIds.has(bundleNodeId)) { let dependencies = dependencyBundleGraph .getNodeIdsConnectedTo( dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), - // $FlowFixMe[incompatible-call] ALL_EDGE_TYPES, ) .map(nodeId => { @@ -147,7 +179,7 @@ function decorateLegacyGraph( bundle = nullthrows( bundleGraph.createBundle({ - entryAsset, + entryAsset: nullthrows(entryAsset), needsStableName: idealBundle.needsStableName, bundleBehavior: idealBundle.bundleBehavior, target: idealBundle.target, @@ -198,7 +230,7 @@ function decorateLegacyGraph( } } - // Step 2: Internalize dependencies for bundles + // Step Internalization: Internalize dependencies for bundles for (let [, idealBundle] of idealBundleGraph.nodes) { if (idealBundle === 'root') continue; let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); @@ -209,6 +241,7 @@ function decorateLegacyGraph( for (let incomingDep of incomingDeps) { if ( incomingDep.priority === 'lazy' && + incomingDep.specifierType !== 'url' && bundle.hasDependency(incomingDep) ) { bundleGraph.internalizeAsyncDependency(bundle, incomingDep); @@ -217,7 +250,7 @@ function decorateLegacyGraph( } } - // Step 3: Add bundles to their bundle groups + // Step Add to BundleGroups: Add bundles to their bundle groups idealBundleGraph.traverse((nodeId, _, actions) => { let node = idealBundleGraph.getNode(nodeId); if (node === 'root') { @@ -242,7 +275,7 @@ function decorateLegacyGraph( } }); - // Step 4: Add references to all bundles + // Step References: Add references to all bundles for (let [asset, references] of idealGraph.assetReference) { for (let [dependency, bundle] of references) { let legacyBundle = nullthrows(idealBundleToLegacyBundle.get(bundle)); @@ -270,6 +303,7 @@ function decorateLegacyGraph( function createIdealGraph( assetGraph: MutableBundleGraph, config: ResolvedBundlerConfig, + entries: Map, ): IdealGraph { // Asset to the bundle and group it's an entry of let bundleRoots: Map = new Map(); @@ -280,35 +314,33 @@ function createIdealGraph( Array<[Dependency, Bundle]>, > = new DefaultMap(() => []); + // A Graph of Bundles and a root node (dummy string), which models only Bundles, and connections to their + // referencing Bundle. There are no actual BundleGroup nodes, just bundles that take on that role. let bundleGraph: Graph = new Graph(); let stack: Array<[BundleRoot, NodeId]> = []; - // bundleGraph that models bundleRoots and async deps only - let asyncBundleRootGraph: ContentGraph = - new ContentGraph(); - let bundleGroupBundleIds: Array = []; + let bundleRootEdgeTypes = { + parallel: 1, + lazy: 2, + }; + // ContentGraph that models bundleRoots, with parallel & async deps only to inform reachability + let bundleRootGraph: ContentGraph< + BundleRoot | 'root', + $Values, + > = new ContentGraph(); - // Step 1: Find and create bundles for entries from assetGraph - let entries: Map = new Map(); - let sharedToSourceBundleIds: Map> = new Map(); + let bundleGroupBundleIds: Set = new Set(); - assetGraph.traverse((node, context, actions) => { - if (node.type !== 'asset') { - return node; - } + // Models bundleRoots and the assets that require it synchronously + let reachableRoots: ContentGraph = new ContentGraph(); - invariant( - context != null && context.type === 'dependency' && context.value.isEntry, - ); - entries.set(node.value, context.value); - actions.skipChildren(); - }); + let sharedToSourceBundleIds: Map> = new Map(); - let rootNodeId = nullthrows(asyncBundleRootGraph.addNode('root')); + let rootNodeId = nullthrows(bundleRootGraph.addNode('root')); let bundleGraphRootNodeId = nullthrows(bundleGraph.addNode('root')); - asyncBundleRootGraph.setRootNodeId(rootNodeId); + bundleRootGraph.setRootNodeId(rootNodeId); bundleGraph.setRootNodeId(bundleGraphRootNodeId); - + // Step Create Entry Bundles for (let [asset, dependency] of entries) { let bundle = createBundle({ asset, @@ -318,9 +350,9 @@ function createIdealGraph( let nodeId = bundleGraph.addNode(bundle); bundles.set(asset.id, nodeId); bundleRoots.set(asset, [nodeId, nodeId]); - asyncBundleRootGraph.addEdge( + bundleRootGraph.addEdge( rootNodeId, - asyncBundleRootGraph.addNodeByContentKey(asset.id, asset), + bundleRootGraph.addNodeByContentKey(asset.id, asset), ); bundleGraph.addEdge(bundleGraphRootNodeId, nodeId); @@ -335,26 +367,38 @@ function createIdealGraph( }), dependencyPriorityEdges[dependency.priority], ); - bundleGroupBundleIds.push(nodeId); + bundleGroupBundleIds.add(nodeId); } let assets = []; - let assetsToAddOnExit: DefaultMap< - Dependency, - Array<[Bundle, Asset]>, - > = new DefaultMap(() => []); - // Step 2: Traverse the asset graph and create bundles for asset type changes and async dependencies, - // only adding the entry asset of each bundle, not the subgraph. + let typeChangeIds = new Set(); + /** + * Step Create Bundles: Traverse the assetGraph (aka MutableBundleGraph) and create bundles + * for asset type changes, parallel, inline, and async or lazy dependencies, + * adding only that asset to each bundle, not its entire subgraph. + */ assetGraph.traverse({ enter(node, context, actions) { if (node.type === 'asset') { + if ( + context?.type === 'dependency' && + context?.value.isEntry && + !entries.has(node.value) + ) { + // Skip whole subtrees of other targets by skipping those entries + actions.skipChildren(); + return node; + } assets.push(node.value); let bundleIdTuple = bundleRoots.get(node.value); - if (bundleIdTuple) { - // Push to the stack when a new bundle is created - stack.push([node.value, bundleIdTuple[1]]); + if (bundleIdTuple && bundleIdTuple[0] === bundleIdTuple[1]) { + // Push to the stack (only) when a new bundle is created + stack.push([node.value, bundleIdTuple[0]]); + } else if (bundleIdTuple) { + // Otherwise, push on the last bundle that marks the start of a BundleGroup + stack.push([node.value, stack[stack.length - 1][1]]); } } else if (node.type === 'dependency') { if (context == null) { @@ -378,7 +422,7 @@ function createIdealGraph( for (let childAsset of assets) { if ( dependency.priority === 'lazy' || - childAsset.bundleBehavior === 'isolated' + childAsset.bundleBehavior === 'isolated' // An isolated Dependency, or Bundle must contain all assets it needs to load. ) { let bundleId = bundles.get(childAsset.id); let bundle; @@ -401,11 +445,20 @@ function createIdealGraph( bundleId = bundleGraph.addNode(bundle); bundles.set(childAsset.id, bundleId); bundleRoots.set(childAsset, [bundleId, bundleId]); - bundleGroupBundleIds.push(bundleId); + bundleGroupBundleIds.add(bundleId); bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); } else { bundle = nullthrows(bundleGraph.getNode(bundleId)); invariant(bundle !== 'root'); + + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } } dependencyBundleGraph.addEdge( @@ -429,72 +482,80 @@ function createIdealGraph( dependency.priority === 'parallel' || childAsset.bundleBehavior === 'inline' ) { - let [, bundleGroupNodeId] = nullthrows(stack[stack.length - 1]); + // The referencing bundleRoot is the root of a Bundle that first brings in another bundle (essentially the FIRST parent of a bundle, this may or may not be a bundleGroup) + let [referencingBundleRoot, bundleGroupNodeId] = nullthrows( + stack[stack.length - 1], + ); let bundleGroup = nullthrows( bundleGraph.getNode(bundleGroupNodeId), ); invariant(bundleGroup !== 'root'); - // Find an existing bundle of the same type within the bundle group. let bundleId; - let entryAsset; - let uniqueKey; + let referencingBundleId = nullthrows( + bundleRoots.get(referencingBundleRoot), + )[0]; + let referencingBundle = nullthrows( + bundleGraph.getNode(referencingBundleId), + ); + invariant(referencingBundle !== 'root'); + let bundle; + bundleId = bundles.get(childAsset.id); + + /** + * If this is an entry bundlegroup, we only allow one bundle per type in those groups + * So attempt to add the asset to the entry bundle if it's of the same type. + * This asset will be created by other dependency if it's in another bundlegroup + * and bundles of other types should be merged in the next step + */ + let bundleGroupRootAsset = nullthrows(bundleGroup.mainEntryAsset); if ( - childAsset.bundleBehavior !== 'inline' && - dependency.priority !== 'parallel' + entries.has(bundleGroupRootAsset) && + bundleGroupRootAsset.type === childAsset.type && + childAsset.bundleBehavior !== 'inline' ) { - uniqueKey = childAsset.id; - // TODO: share bundles even across different bundle groups by looking if the child - // asset is already a bundle root. In order for this to work, bundleRoots must be - // keyed by asset + target, not just asset, so that bundles are not shared between targets. - bundleId = - bundleGroup.type == childAsset.type - ? bundleGroupNodeId - : bundleGraph - .getNodeIdsConnectedFrom(bundleGroupNodeId) - .find(id => { - let node = bundleGraph.getNode(id); - return node !== 'root' && node?.type == childAsset.type; - }); - } else { - entryAsset = childAsset; + bundleId = bundleGroupNodeId; } - - let bundle; if (bundleId == null) { - // Create a new bundle if none of the same type exists already. bundle = createBundle({ - // We either have an entry asset or a unique key. // Bundles created from type changes shouldn't have an entry asset. - asset: entryAsset, - uniqueKey, + asset: childAsset, type: childAsset.type, env: childAsset.env, bundleBehavior: childAsset.bundleBehavior, - target: bundleGroup.target, + target: referencingBundle.target, needsStableName: childAsset.bundleBehavior === 'inline' || dependency.bundleBehavior === 'inline' || (dependency.priority === 'parallel' && !dependency.needsStableName) ? false - : bundleGroup.needsStableName, + : referencingBundle.needsStableName, }); bundleId = bundleGraph.addNode(bundle); + // Store Type-Change bundles for later since we need to know ALL bundlegroups they are part of to reduce/combine them + typeChangeIds.add(bundleId); + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } } else { - // Otherwise, merge this asset into the existing bundle. + // Otherwise, merge. bundle = bundleGraph.getNode(bundleId); invariant(bundle != null && bundle !== 'root'); } - if (!entryAsset) { - // Queue the asset to be added on exit of this node, so we add dependencies first. - assetsToAddOnExit.get(dependency).push([bundle, childAsset]); - } - bundles.set(childAsset.id, bundleId); + + // A bundle can belong to multiple bundlegroups, all the bundle groups of it's + // ancestors, and all async and entry bundles before it are "bundle groups" + // TODO: We may need to track bundles to all bundleGroups it belongs to in the future. bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); + bundleGraph.addEdge(referencingBundleId, bundleId); if (bundleId != bundleGroupNodeId) { dependencyBundleGraph.addEdge( @@ -514,10 +575,6 @@ function createIdealGraph( ), dependencyPriorityEdges.parallel, ); - - // Add an edge from the bundle group entry to the new bundle. - // This indicates that the bundle is loaded together with the entry - bundleGraph.addEdge(bundleGroupNodeId, bundleId); } assetReference.get(childAsset).push([dependency, bundle]); @@ -528,32 +585,69 @@ function createIdealGraph( return node; }, exit(node) { - if (node.type === 'dependency' && assetsToAddOnExit.has(node.value)) { - let assetsToAdd = assetsToAddOnExit.get(node.value); - for (let [bundle, asset] of assetsToAdd) { - bundle.assets.add(asset); - bundle.size += asset.stats.size; - } - assetsToAddOnExit.delete(node.value); - } - if (stack[stack.length - 1]?.[0] === node.value) { stack.pop(); } }, }); - // Step 3: Determine reachability for every asset from each bundleRoot. - // This is later used to determine which bundles to place each asset in. + // Step Merge Type Change Bundles: Clean up type change bundles within the exact same bundlegroups + for (let [nodeIdA, a] of bundleGraph.nodes) { + //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes + if (!typeChangeIds.has(nodeIdA) || a === 'root') continue; + let bundleABundleGroups = getBundleGroupsForBundle(nodeIdA); + for (let [nodeIdB, b] of bundleGraph.nodes) { + if ( + a !== 'root' && + b !== 'root' && + a !== b && + typeChangeIds.has(nodeIdB) && + a.bundleBehavior !== 'inline' && + b.bundleBehavior !== 'inline' && + a.type === b.type + ) { + let bundleBbundleGroups = getBundleGroupsForBundle(nodeIdB); + if (setEqual(bundleBbundleGroups, bundleABundleGroups)) { + let shouldMerge = true; + for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(nodeIdB)), + ALL_EDGE_TYPES, + )) { + let depNode = dependencyBundleGraph.getNode(depId); + // Cannot merge Dependency URL specifier type + if ( + depNode && + depNode.type === 'dependency' && + depNode.value.specifierType === 'url' + ) { + shouldMerge = false; + continue; + } + } + if (!shouldMerge) continue; + mergeBundle(nodeIdA, nodeIdB); + } + } + } + } + + /** + * Step Determine Reachability: Determine reachability for every asset from each bundleRoot. + * This is later used to determine which bundles to place each asset in. We build up two + * structures, one traversal each. ReachableRoots to store sync relationships, + * and bundleRootGraph to store the minimal availability through `parallel` and `async` relationships. + * The two graphs, are used to build up ancestorAssets, a structure which holds all availability by + * all means for each asset. + */ for (let [root] of bundleRoots) { if (!entries.has(root)) { - asyncBundleRootGraph.addNodeByContentKey(root.id, root); + bundleRootGraph.addNodeByContentKey(root.id, root); // Add in all bundleRoots to BundleRootGraph } } - - // Models bundleRoots and the assets that require it synchronously - let reachableRoots: ContentGraph = new ContentGraph(); + // ReachableRoots is a Graph of Asset Nodes which represents a BundleRoot, to all assets (non-bundleroot assets + // available to it synchronously (directly) built by traversing the assetgraph once. for (let [root] of bundleRoots) { + // Add sync relationships to ReachableRoots let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); assetGraph.traverse((node, _, actions) => { if (node.value === root) { @@ -576,13 +670,15 @@ function createIdealGraph( ); if ( bundle !== 'root' && - bundle.bundleBehavior !== 'isolated' && bundle.bundleBehavior !== 'inline' && !bundle.env.isIsolated() ) { - asyncBundleRootGraph.addEdge( - asyncBundleRootGraph.getNodeIdByContentKey(root.id), - asyncBundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + bundleRootGraph.addEdge( + bundleRootGraph.getNodeIdByContentKey(root.id), + bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + dependency.priority === 'parallel' + ? bundleRootEdgeTypes.parallel + : bundleRootEdgeTypes.lazy, ); } } @@ -613,29 +709,40 @@ function createIdealGraph( // Maps a given bundleRoot to the assets reachable from it, // and the bundleRoots reachable from each of these assets - let asyncAncestorAssets: Map> = new Map(); + let ancestorAssets: Map> = new Map(); - // Step 4: Determine assets that should be duplicated by computing asset availability in each bundle group for (let entry of entries.keys()) { // Initialize an empty set of ancestors available to entries - asyncAncestorAssets.set(entry, new Set()); + ancestorAssets.set(entry, new Set()); } + // Step Determine Availability // Visit nodes in a topological order, visiting parent nodes before child nodes. + // This allows us to construct an understanding of which assets will already be // loaded and available when a bundle runs, by pushing available assets downwards and // computing the intersection of assets available through all possible paths to a bundle. - for (let nodeId of asyncBundleRootGraph.topoSort()) { - const bundleRoot = asyncBundleRootGraph.getNode(nodeId); + // We call this structure ancestorAssets, a Map that tracks a bundleRoot, + // to all assets available to it (meaning they will exist guaranteed when the bundleRoot is loaded) + // The topological sort ensures all parents are visited before the node we want to process. + for (let nodeId of bundleRootGraph.topoSort(ALL_EDGE_TYPES)) { + const bundleRoot = bundleRootGraph.getNode(nodeId); if (bundleRoot === 'root') continue; invariant(bundleRoot != null); let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; + // At a BundleRoot, we access it's available assets (via ancestorAssets), + // and add to that all assets within the bundles in that BundleGroup. + + // This set is available to all bundles in a particular bundleGroup because + // bundleGroups are just bundles loaded at the same time. However it is + // not true that a bundle's available assets = all assets of all the bundleGroups + // it belongs to. It's the intersection of those sets. let available; if (bundleRoot.bundleBehavior === 'isolated') { available = new Set(); } else { - available = new Set(asyncAncestorAssets.get(bundleRoot)); + available = new Set(ancestorAssets.get(bundleRoot)); for (let bundleIdInGroup of [ bundleGroupId, ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), @@ -664,29 +771,95 @@ function createIdealGraph( } } - let children = asyncBundleRootGraph.getNodeIdsConnectedFrom(nodeId); - // Group assets available across our children by the child. This will be used - // to determine borrowers if needed below. + // Now that we have bundleGroup availability, we will propagate that down to all the children + // of this bundleGroup. For a child, we also must maintain parallel availability. If it has + // parallel siblings that come before it, those, too, are available to it. Add those parallel + // available assets to the set of available assets for this child as well. + let children = bundleRootGraph.getNodeIdsConnectedFrom( + nodeId, + ALL_EDGE_TYPES, + ); + let parallelAvailability: Set = new Set(); + for (let childId of children) { - let child = asyncBundleRootGraph.getNode(childId); + let child = bundleRootGraph.getNode(childId); invariant(child !== 'root' && child != null); - if ( - child.bundleBehavior === 'isolated' || - child.bundleBehavior === 'inline' - ) { + let bundleBehavior = getBundleFromBundleRoot(child).bundleBehavior; + if (bundleBehavior === 'isolated' || bundleBehavior === 'inline') { continue; } + let isParallel = bundleRootGraph.hasEdge( + nodeId, + childId, + bundleRootEdgeTypes.parallel, + ); - const childAvailableAssets = asyncAncestorAssets.get(child); + // Most of the time, a child will have many parent bundleGroups, + // so the next time we peek at a child from another parent, we will + // intersect the availability built there with the previously computed + // availability. this ensures no matter which bundleGroup loads a particular bundle, + // it will only assume availability of assets it has under any circumstance + const childAvailableAssets = ancestorAssets.get(child); + let currentChildAvailable = isParallel + ? setUnion(parallelAvailability, available) + : available; if (childAvailableAssets != null) { - setIntersect(childAvailableAssets, available); + setIntersect(childAvailableAssets, currentChildAvailable); } else { - asyncAncestorAssets.set(child, new Set(available)); + ancestorAssets.set(child, new Set(currentChildAvailable)); + } + if (isParallel) { + let assetsFromBundleRoot = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(child.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); + parallelAvailability = setUnion( + parallelAvailability, + assetsFromBundleRoot, + ); } } } + // Step Internalize async bundles - internalize Async bundles if and only if, + // the bundle is synchronously available elsewhere. + // We can query sync assets available via reachableRoots. If the parent has + // the bundleRoot by reachableRoots AND ancestorAssets, internalize it. + for (let [id, bundleRoot] of bundleRootGraph.nodes) { + if (bundleRoot === 'root') continue; + let parentRoots = bundleRootGraph + .getNodeIdsConnectedTo(id, ALL_EDGE_TYPES) + .map(id => nullthrows(bundleRootGraph.getNode(id))); + let canDelete = + getBundleFromBundleRoot(bundleRoot).bundleBehavior !== 'isolated'; + if (parentRoots.length === 0) continue; + for (let parent of parentRoots) { + if (parent === 'root') { + canDelete = false; + continue; + } + if ( + reachableRoots.hasEdge( + reachableRoots.getNodeIdByContentKey(parent.id), + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + ) || + ancestorAssets.get(parent)?.has(bundleRoot) + ) { + let parentBundle = bundleGraph.getNode( + nullthrows(bundles.get(parent.id)), + ); + invariant(parentBundle != null && parentBundle !== 'root'); + parentBundle.internalizedAssetIds.push(bundleRoot.id); + } else { + canDelete = false; + } + } + if (canDelete) { + deleteBundle(bundleRoot); + } + } - // Step 5: Place all assets into bundles or create shared bundles. Each asset + // Step Insert Or Share: Place all assets into bundles or create shared bundles. Each asset // is placed into a single bundle based on the bundle entries it is reachable from. // This creates a maximally code split bundle graph with no duplication. for (let asset of assets) { @@ -696,88 +869,279 @@ function createIdealGraph( reachableRoots, ).reverse(); + let reachableEntries = []; + let reachableNonEntries = []; + + // Filter out entries, since they can't have shared bundles. + // Neither can non-splittable, isolated, or needing of stable name bundles. + // Reserve those filtered out bundles since we add the asset back into them. + for (let a of reachable) { + if ( + entries.has(a) || + !a.isBundleSplittable || + getBundleFromBundleRoot(a).needsStableName || + getBundleFromBundleRoot(a).bundleBehavior === 'isolated' + ) { + reachableEntries.push(a); + } else { + reachableNonEntries.push(a); + } + } + reachable = reachableNonEntries; + // Filter out bundles from this asset's reachable array if // bundle does not contain the asset in its ancestry - reachable = reachable.filter(b => !asyncAncestorAssets.get(b)?.has(asset)); + reachable = reachable.filter(b => !ancestorAssets.get(b)?.has(asset)); + + // Finally, filter out bundleRoots (bundles) from this assets + // reachable if they are subgraphs, and reuse that subgraph bundle + // by drawing an edge. Essentially, if two bundles within an assets + // reachable array, have an ancestor-subgraph relationship, draw that edge. + // This allows for us to reuse a bundle instead of making a shared bundle if + // a bundle represents the exact set of assets a set of bundles would share + + // if a bundle b is a subgraph of another bundle f, reuse it, drawing an edge between the two + reachable = reachable.filter(b => { + if (b.env.isIsolated()) { + return true; + } + let toKeep = true; + if (bundles.has(asset.id)) { + toKeep = false; + bundleGraph.addEdge( + nullthrows(bundles.get(b.id)), + nullthrows(bundles.get(asset.id)), + ); + } + for (let f of reachable) { + if (b === f) continue; + let fReachable = getReachableBundleRoots(f, reachableRoots).filter( + b => !ancestorAssets.get(b)?.has(f), + ); + if (fReachable.indexOf(b) > -1) { + toKeep = false; + bundleGraph.addEdge( + nullthrows(bundles.get(b.id)), + nullthrows(bundles.get(f.id)), + ); + } + } + return toKeep; + }); - if (reachable.length > 0) { - let reachableEntries = reachable.filter( - a => - entries.has(a) || - !a.isBundleSplittable || - getBundleFromBundleRoot(a).needsStableName || - getBundleFromBundleRoot(a).bundleBehavior === 'inline' || - getBundleFromBundleRoot(a).bundleBehavior === 'isolated', - ); - reachable = reachable.filter( - a => - !entries.has(a) && - a.isBundleSplittable && - !getBundleFromBundleRoot(a).needsStableName && - getBundleFromBundleRoot(a).bundleBehavior !== 'inline' && - getBundleFromBundleRoot(a).bundleBehavior !== 'isolated', - ); + // Add assets to non-splittable bundles. + for (let entry of reachableEntries) { + let entryBundleId = nullthrows(bundles.get(entry.id)); + let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); + invariant(entryBundle !== 'root'); + entryBundle.assets.add(asset); + entryBundle.size += asset.stats.size; + } - // Add assets to non-splittable bundles. - for (let entry of reachableEntries) { - let bundleId = nullthrows(bundles.get(entry.id)); - let bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); - bundle.assets.add(asset); - bundle.size += asset.stats.size; - } + // Create shared bundles for splittable bundles. + if (reachable.length > 0) { + let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); + let key = reachable.map(a => a.id).join(','); + let bundleId = bundles.get(key); + let bundle; + if (bundleId == null) { + let firstSourceBundle = nullthrows( + bundleGraph.getNode(sourceBundles[0]), + ); + invariant(firstSourceBundle !== 'root'); + bundle = createBundle({ + target: firstSourceBundle.target, + type: firstSourceBundle.type, + env: firstSourceBundle.env, + }); + bundle.sourceBundles = sourceBundles; + let sharedInternalizedAssets = new Set( + firstSourceBundle.internalizedAssetIds, + ); - // Create shared bundles for splittable bundles. - if (reachable.length > 0) { - let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); - let key = reachable.map(a => a.id).join(','); - let bundleId = bundles.get(key); - let bundle; - if (bundleId == null) { - let firstSourceBundle = nullthrows( - bundleGraph.getNode(sourceBundles[0]), + for (let p of sourceBundles) { + let parentBundle = nullthrows(bundleGraph.getNode(p)); + invariant(parentBundle !== 'root'); + if (parentBundle === firstSourceBundle) continue; + setIntersect( + sharedInternalizedAssets, + new Set(parentBundle.internalizedAssetIds), ); - invariant(firstSourceBundle !== 'root'); - bundle = createBundle({ - target: firstSourceBundle.target, - type: firstSourceBundle.type, - env: firstSourceBundle.env, - }); - bundle.sourceBundles = sourceBundles; - bundleId = bundleGraph.addNode(bundle); - bundles.set(key, bundleId); - } else { - bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); } - bundle.assets.add(asset); - bundle.size += asset.stats.size; + bundle.internalizedAssetIds = [...sharedInternalizedAssets]; + bundleId = bundleGraph.addNode(bundle); + bundles.set(key, bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); + } + bundle.assets.add(asset); + bundle.size += asset.stats.size; - for (let sourceBundleId of sourceBundles) { - if (bundleId !== sourceBundleId) { - bundleGraph.addEdge(sourceBundleId, bundleId); - } + for (let sourceBundleId of sourceBundles) { + if (bundleId !== sourceBundleId) { + bundleGraph.addEdge(sourceBundleId, bundleId); } - sharedToSourceBundleIds.set(bundleId, sourceBundles); - - dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { - value: bundle, - type: 'bundle', - }); } + sharedToSourceBundleIds.set(bundleId, sourceBundles); + + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { + value: bundle, + type: 'bundle', + }); } } - // Step 7: Merge any shared bundles under the minimum bundle size back into + // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into // their source bundles, and remove the bundle. for (let [bundleNodeId, bundle] of bundleGraph.nodes) { if (bundle === 'root') continue; if (bundle.sourceBundles.length > 0 && bundle.size < config.minBundleSize) { sharedToSourceBundleIds.delete(bundleNodeId); - removeBundle(bundleGraph, bundleNodeId); + removeBundle(bundleGraph, bundleNodeId, assetReference); + } + } + + // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit. + for (let [bundleId, bundleGroupId] of bundleRoots.values()) { + // Only handle bundle group entries. + if (bundleId !== bundleGroupId) { + continue; + } + + // Find shared bundles in this bundle group. + let bundleIdsInGroup = []; + for (let [ + sharedBundleId, + sourceBundleIds, + ] of sharedToSourceBundleIds.entries()) { + // If the bundle group's entry is a source bundle of this shared bundle, + // the shared bundle is part of the bundle group. + if (sourceBundleIds.includes(bundleId)) { + bundleIdsInGroup.push(sharedBundleId); + } + } + + if (bundleIdsInGroup.length > config.maxParallelRequests) { + // Sort the bundles so the smallest ones are removed first. + let bundlesInGroup = bundleIdsInGroup + .map(id => ({ + id, + bundle: nullthrows(bundleGraph.getNode(id)), + })) + .map(({id, bundle}) => { + // For Flow + invariant(bundle !== 'root'); + return {id, bundle}; + }) + .sort((a, b) => a.bundle.size - b.bundle.size); + + // Remove bundles until the bundle group is within the parallel request limit. + for ( + let i = 0; + i < bundlesInGroup.length - config.maxParallelRequests; + i++ + ) { + let bundleToRemove = bundlesInGroup[i].bundle; + let bundleIdToRemove = bundlesInGroup[i].id; + + // Add all assets in the shared bundle into the source bundles that are within this bundle group. + let sourceBundles = bundleToRemove.sourceBundles + .filter(b => bundlesInGroup.map(b => b.bundle).includes(b)) + .map(id => nullthrows(bundleGraph.getNode(id))); + + for (let sourceBundle of sourceBundles) { + invariant(sourceBundle !== 'root'); + for (let asset of bundleToRemove.assets) { + sourceBundle.assets.add(asset); + sourceBundle.size += asset.stats.size; + } + } + + // Remove the edge from this bundle group to the shared bundle. + bundleGraph.removeEdge(bundleGroupId, bundleIdToRemove); + + // If there is now only a single bundle group that contains this bundle, + // merge it into the remaining source bundles. If it is orphaned entirely, remove it. + let incomingNodeCount = + bundleGraph.getNodeIdsConnectedTo(bundleIdToRemove).length; + if (incomingNodeCount === 1) { + removeBundle(bundleGraph, bundleIdToRemove, assetReference); + for (let sharedBundleId of sharedToSourceBundleIds.keys()) { + if (sharedBundleId === bundleIdToRemove) { + sharedToSourceBundleIds.delete(sharedBundleId); + } + } + } else if (incomingNodeCount === 0) { + bundleGraph.removeNode(bundleIdToRemove); + } + } + } + } + + function deleteBundle(bundleRoot: BundleRoot) { + bundleGraph.removeNode(nullthrows(bundles.get(bundleRoot.id))); + bundleRoots.delete(bundleRoot); + bundles.delete(bundleRoot.id); + if (reachableRoots.hasContentKey(bundleRoot.id)) { + reachableRoots.replaceNodeIdsConnectedTo( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + [], + ); } + if (bundleRootGraph.hasContentKey(bundleRoot.id)) { + bundleRootGraph.removeNode( + bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + ); + } + } + function getBundleGroupsForBundle(nodeId: NodeId) { + let bundleGroupBundleIds = new Set(); + bundleGraph.traverseAncestors(nodeId, ancestorId => { + if ( + bundleGraph + .getNodeIdsConnectedTo(ancestorId) + .includes(bundleGraph.rootNodeId) + ) { + bundleGroupBundleIds.add(ancestorId); + } + }); + return bundleGroupBundleIds; } + function mergeBundle(mainNodeId: NodeId, otherNodeId: NodeId) { + //merges assets of "otherRoot" into "mainBundleRoot" + let a = nullthrows(bundleGraph.getNode(mainNodeId)); + let b = nullthrows(bundleGraph.getNode(otherNodeId)); + invariant(a !== 'root' && b !== 'root'); + let bundleRootB = nullthrows(b.mainEntryAsset); + let mainBundleRoot = nullthrows(a.mainEntryAsset); + for (let asset of a.assets) { + b.assets.add(asset); + } + a.assets = b.assets; + for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(otherNodeId)), + ALL_EDGE_TYPES, + )) { + dependencyBundleGraph.replaceNodeIdsConnectedTo(depId, [ + dependencyBundleGraph.getNodeIdByContentKey(String(mainNodeId)), + ]); + } + + //clean up asset reference + for (let dependencyTuple of assetReference.get(bundleRootB)) { + dependencyTuple[1] = a; + } + //add in any lost edges + for (let nodeId of bundleGraph.getNodeIdsConnectedTo(otherNodeId)) { + bundleGraph.addEdge(nodeId, mainNodeId); + } + deleteBundle(bundleRootB); + let bundleGroupOfMain = nullthrows(bundleRoots.get(mainBundleRoot))[1]; + bundleRoots.set(bundleRootB, [mainNodeId, bundleGroupOfMain]); + bundles.set(bundleRootB.id, mainNodeId); + } function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { let bundle = bundleGraph.getNode( nullthrows(bundleRoots.get(bundleRoot))[0], @@ -856,11 +1220,18 @@ function createBundle(opts: {| }; } -function removeBundle(bundleGraph: Graph, bundleId: NodeId) { +function removeBundle( + bundleGraph: Graph, + bundleId: NodeId, + assetReference: DefaultMap>, +) { let bundle = nullthrows(bundleGraph.getNode(bundleId)); invariant(bundle !== 'root'); - for (let asset of bundle.assets) { + assetReference.set( + asset, + assetReference.get(asset).filter(t => !t.includes(bundle)), + ); for (let sourceBundleId of bundle.sourceBundles) { let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); invariant(sourceBundle !== 'root'); @@ -879,6 +1250,7 @@ async function loadBundlerConfig( let conf = await config.getConfig([], { packageKey: '@parcel/bundler-default', }); + if (!conf) { return HTTP_OPTIONS['2']; } @@ -893,8 +1265,8 @@ async function loadBundlerConfig( filePath: conf.filePath, prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-default')}`, }, - '@parcel/bundler-default', - 'Invalid config for @parcel/bundler-default', + '@parcel/bundler-experimental', + 'Invalid config for @parcel/bundler-experimental', ); let http = conf.contents.http ?? 2; @@ -913,3 +1285,29 @@ function getReachableBundleRoots(asset, graph): Array { .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) .map(nodeId => nullthrows(graph.getNode(nodeId))); } + +function getEntryByTarget( + bundleGraph: MutableBundleGraph, +): DefaultMap> { + // Find entries from assetGraph per target + let targets: DefaultMap> = new DefaultMap( + () => new Map(), + ); + bundleGraph.traverse({ + enter(node, context, actions) { + if (node.type !== 'asset') { + return node; + } + invariant( + context != null && + context.type === 'dependency' && + context.value.isEntry && + context.value.target != null, + ); + targets.get(context.value.target.distDir).set(node.value, context.value); + actions.skipChildren(); + return node; + }, + }); + return targets; +} diff --git a/packages/core/core/src/BundleGraph.js b/packages/core/core/src/BundleGraph.js index 4ee291557fd..b0ab7d12864 100644 --- a/packages/core/core/src/BundleGraph.js +++ b/packages/core/core/src/BundleGraph.js @@ -732,12 +732,13 @@ export default class BundleGraph { if ( inboundDependencies.every( dependency => - !this.bundleHasDependency(bundle, dependency) || - this._graph.hasEdge( - bundleNodeId, - this._graph.getNodeIdByContentKey(dependency.id), - bundleGraphEdgeTypes.internal_async, - ), + dependency.specifierType !== SpecifierType.url && + (!this.bundleHasDependency(bundle, dependency) || + this._graph.hasEdge( + bundleNodeId, + this._graph.getNodeIdByContentKey(dependency.id), + bundleGraphEdgeTypes.internal_async, + )), ) ) { this._graph.removeEdge( @@ -1307,9 +1308,10 @@ export default class BundleGraph { // Shared bundles seem to depend on being used in the opposite order // they were added. // TODO: Should this be the case? - this._graph - .getNodeIdsConnectedFrom(nodeId, bundleGraphEdgeTypes.references) - .reverse(), + this._graph.getNodeIdsConnectedFrom( + nodeId, + bundleGraphEdgeTypes.references, + ), }); return [...referencedBundles]; diff --git a/packages/core/graph/src/Graph.js b/packages/core/graph/src/Graph.js index 6a3b5b64466..31faaf3b7a1 100644 --- a/packages/core/graph/src/Graph.js +++ b/packages/core/graph/src/Graph.js @@ -407,13 +407,17 @@ export default class Graph { return null; } - topoSort(): Array { + topoSort(type?: TEdgeType): Array { let sorted: Array = []; - this.traverse({ - exit: nodeId => { - sorted.push(nodeId); + this.traverse( + { + exit: nodeId => { + sorted.push(nodeId); + }, }, - }); + null, + type, + ); return sorted.reverse(); } diff --git a/packages/core/integration-tests/test/cache.js b/packages/core/integration-tests/test/cache.js index b72cc66eefd..3cfc1b01f3d 100644 --- a/packages/core/integration-tests/test/cache.js +++ b/packages/core/integration-tests/test/cache.js @@ -4360,7 +4360,7 @@ describe('cache', function () { it('should support adding bundler config', async function () { let b = await testCache( { - entries: ['*.html'], + entries: ['index.js'], mode: 'production', async setup() { let pkgFile = path.join(inputDir, 'package.json'); @@ -4374,13 +4374,28 @@ describe('cache', function () { ); }, async update(b) { - let html = await overlayFS.readFile( - b.bundleGraph.getBundles().find(b => b.name === 'b.html') - ?.filePath, - 'utf8', - ); - assert.equal(html.match(/