diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0522861b282..dbac6eb54da 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -96,46 +96,11 @@ jobs: name: Integration tests (${{matrix.os}}, node ${{matrix.node}}) path: "**/junit-*.xml" - experimental_bundler_integration_tests: - name: Experimental Bundler Integration tests (${{matrix.os}}, Node ${{matrix.node}}) - strategy: - matrix: - node: [14, 16] - os: [ubuntu-latest, macos-latest, windows-latest] - # These tend to be quite flakey, so one failed instance shouldn't stop - # others from potentially succeeding - fail-fast: false - runs-on: ${{matrix.os}} - steps: - - uses: actions/checkout@v2 - - uses: actions/setup-node@v2 - with: - cache: yarn - node-version: ${{matrix.node}} - - uses: actions-rs/toolchain@v1 - - uses: Swatinem/rust-cache@v1 - - name: Bump max inotify watches (Linux only) - run: echo fs.inotify.max_user_watches=524288 | sudo tee -a /etc/sysctl.conf && sudo sysctl -p; - if: ${{matrix.os == 'ubuntu-latest'}} - - run: yarn --frozen-lockfile - - run: yarn build-native-release - - run: yarn test:integration-ci - env: - PARCEL_TEST_EXPERIMENTAL_BUNDLER: 1 - # Similar to - # https://github.com/marketplace/actions/publish-unit-test-results#use-with-matrix-strategy - - name: Upload JUnit results - if: always() - uses: actions/upload-artifact@v2 - with: - name: Exprimental Bundler Integration tests (${{matrix.os}}, node ${{matrix.node}}) - path: "**/junit-*.xml" - test_report: name: Test report runs-on: ubuntu-latest needs: - [unit_tests, integration_tests, experimental_bundler_integration_tests] + [unit_tests, integration_tests] if: always() steps: - name: Create test report diff --git a/packages/bundlers/default/package.json b/packages/bundlers/default/package.json index f36f3b1af45..29c1cf48d4a 100644 --- a/packages/bundlers/default/package.json +++ b/packages/bundlers/default/package.json @@ -21,6 +21,7 @@ }, "dependencies": { "@parcel/diagnostic": "2.7.0", + "@parcel/graph": "2.7.0", "@parcel/hash": "2.7.0", "@parcel/plugin": "2.7.0", "@parcel/utils": "2.7.0", diff --git a/packages/bundlers/default/src/DefaultBundler.js b/packages/bundlers/default/src/DefaultBundler.js index 6feb5c6fe6e..9d1bf240c71 100644 --- a/packages/bundlers/default/src/DefaultBundler.js +++ b/packages/bundlers/default/src/DefaultBundler.js @@ -2,19 +2,30 @@ import type { Asset, - Bundle, + Bundle as LegacyBundle, + BundleBehavior, BundleGroup, + Dependency, + Environment, Config, MutableBundleGraph, PluginOptions, - Dependency, + Target, } from '@parcel/types'; +import type {NodeId} from '@parcel/graph'; import type {SchemaEntity} from '@parcel/utils'; +import {ContentGraph, Graph} from '@parcel/graph'; import invariant from 'assert'; +import {ALL_EDGE_TYPES} from '@parcel/graph'; import {Bundler} from '@parcel/plugin'; -import {validateSchema, DefaultMap} from '@parcel/utils'; -import {hashString} from '@parcel/hash'; +import { + setIntersect, + setUnion, + setEqual, + validateSchema, + DefaultMap, +} from '@parcel/utils'; import nullthrows from 'nullthrows'; import {encodeJSONKeyComponent} from '@parcel/diagnostic'; @@ -25,6 +36,12 @@ type BundlerConfig = {| maxParallelRequests?: number, |}; +type ResolvedBundlerConfig = {| + minBundles: number, + minBundleSize: number, + maxParallelRequests: number, +|}; + // Default options by http version. const HTTP_OPTIONS = { '1': { @@ -39,401 +56,1145 @@ const HTTP_OPTIONS = { }, }; -let skipOptimize = false; +type AssetId = string; -export default (new Bundler({ - // RULES: - // 1. If dep.isAsync or dep.isEntry, start a new bundle group. - // 2. If an asset is a different type than the current bundle, make a parallel bundle in the same bundle group. - // 3. If an asset is already in a parent bundle in the same entry point, exclude from child bundles. - // 4. If an asset is only in separate isolated entry points (e.g. workers, different HTML pages), duplicate it. +/* BundleRoot - An asset that is the main entry of a Bundle. */ +type BundleRoot = Asset; +export type Bundle = {| + uniqueKey: ?string, + assets: Set, + internalizedAssetIds: Array, + bundleBehavior?: ?BundleBehavior, + needsStableName: boolean, + mainEntryAsset: ?Asset, + size: number, + sourceBundles: Set, + target: Target, + env: Environment, + type: string, +|}; + +const dependencyPriorityEdges = { + sync: 1, + parallel: 2, + lazy: 3, +}; + +type DependencyBundleGraph = ContentGraph< + | {| + value: Bundle, + type: 'bundle', + |} + | {| + value: Dependency, + type: 'dependency', + |}, + number, +>; +// IdealGraph is the structure we will pass to decorate, +// which mutates the assetGraph into the bundleGraph we would +// expect from default bundler +type IdealGraph = {| + dependencyBundleGraph: DependencyBundleGraph, + bundleGraph: Graph, + bundleGroupBundleIds: Set, + assetReference: DefaultMap>, +|}; +/** + * + * The Bundler works by creating an IdealGraph, which contains a BundleGraph that models bundles + * connected to othervbundles by what references them, and thus models BundleGroups. + * + * First, we enter `bundle({bundleGraph, config})`. Here, "bundleGraph" is actually just the + * assetGraph turned into a type `MutableBundleGraph`, which will then be mutated in decorate, + * and turned into what we expect the bundleGraph to be as per the old (default) bundler structure + * & what the rest of Parcel expects a BundleGraph to be. + * + * `bundle({bundleGraph, config})` First gets a Mapping of target to entries, In most cases there is + * only one target, and one or more entries. (Targets are pertinent in monorepos or projects where you + * will have two or more distDirs, or output folders.) Then calls create IdealGraph and Decorate per target. + * + */ +export default (new Bundler({ loadConfig({config, options}) { return loadBundlerConfig(config, options); }, bundle({bundleGraph, config}) { - let bundleRoots: Map> = new Map(); - let bundlesByEntryAsset: Map = new Map(); - let assetsToAddOnExit: DefaultMap< - Dependency, - Array<[Bundle, Asset]>, - > = new DefaultMap(() => []); - - // Step 1: create bundles for each of the explicit code split points. - bundleGraph.traverse({ - enter: (node, context, actions) => { - if (node.type !== 'dependency') { - return { - ...context, - bundleGroup: context?.bundleGroup, - bundleByType: context?.bundleByType, - parentNode: node, - parentBundle: - bundlesByEntryAsset.get(node.value) ?? context?.parentBundle, - }; + let targetMap = getEntryByTarget(bundleGraph); // Organize entries by target output folder/ distDir + let graphs = []; + for (let entries of targetMap.values()) { + // Create separate bundleGraphs per distDir + graphs.push(createIdealGraph(bundleGraph, config, entries)); + } + for (let g of graphs) { + decorateLegacyGraph(g, bundleGraph); //mutate original graph + } + }, + optimize() {}, +}): Bundler); + +function decorateLegacyGraph( + idealGraph: IdealGraph, + bundleGraph: MutableBundleGraph, +): void { + let idealBundleToLegacyBundle: Map = new Map(); + + let { + bundleGraph: idealBundleGraph, + dependencyBundleGraph, + bundleGroupBundleIds, + } = idealGraph; + let entryBundleToBundleGroup: Map = new Map(); + // Step Create Bundles: Create bundle groups, bundles, and shared bundles and add assets to them + for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { + if (idealBundle === 'root') continue; + let entryAsset = idealBundle.mainEntryAsset; + let bundleGroup; + let bundle; + + if (bundleGroupBundleIds.has(bundleNodeId)) { + let dependencies = dependencyBundleGraph + .getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), + ALL_EDGE_TYPES, + ) + .map(nodeId => { + let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); + invariant(dependency.type === 'dependency'); + return dependency.value; + }); + for (let dependency of dependencies) { + bundleGroup = bundleGraph.createBundleGroup( + dependency, + idealBundle.target, + ); + } + invariant(bundleGroup); + entryBundleToBundleGroup.set(bundleNodeId, bundleGroup); + + bundle = nullthrows( + bundleGraph.createBundle({ + entryAsset: nullthrows(entryAsset), + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + target: idealBundle.target, + }), + ); + + bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); + } else if (idealBundle.sourceBundles.size > 0) { + bundle = nullthrows( + bundleGraph.createBundle({ + uniqueKey: + [...idealBundle.assets].map(asset => asset.id).join(',') + + [...idealBundle.sourceBundles].join(','), + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + type: idealBundle.type, + target: idealBundle.target, + env: idealBundle.env, + }), + ); + } else if (idealBundle.uniqueKey != null) { + bundle = nullthrows( + bundleGraph.createBundle({ + uniqueKey: idealBundle.uniqueKey, + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + type: idealBundle.type, + target: idealBundle.target, + env: idealBundle.env, + }), + ); + } else { + invariant(entryAsset != null); + bundle = nullthrows( + bundleGraph.createBundle({ + entryAsset, + needsStableName: idealBundle.needsStableName, + bundleBehavior: idealBundle.bundleBehavior, + target: idealBundle.target, + }), + ); + } + + idealBundleToLegacyBundle.set(idealBundle, bundle); + + for (let asset of idealBundle.assets) { + bundleGraph.addAssetToBundle(asset, bundle); + } + } + // Step Internalization: Internalize dependencies for bundles + for (let [, idealBundle] of idealBundleGraph.nodes) { + if (idealBundle === 'root') continue; + let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); + for (let internalized of idealBundle.internalizedAssetIds) { + let incomingDeps = bundleGraph.getIncomingDependencies( + bundleGraph.getAssetById(internalized), + ); + for (let incomingDep of incomingDeps) { + if ( + incomingDep.priority === 'lazy' && + incomingDep.specifierType !== 'url' && + bundle.hasDependency(incomingDep) + ) { + bundleGraph.internalizeAsyncDependency(bundle, incomingDep); } + } + } + } + + // Step Add to BundleGroups: Add bundles to their bundle groups + idealBundleGraph.traverse((nodeId, _, actions) => { + let node = idealBundleGraph.getNode(nodeId); + if (node === 'root') { + return; + } + actions.skipChildren(); + + let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(nodeId); + let entryBundle = nullthrows(idealBundleGraph.getNode(nodeId)); + invariant(entryBundle !== 'root'); + let legacyEntryBundle = nullthrows( + idealBundleToLegacyBundle.get(entryBundle), + ); + + for (let id of outboundNodeIds) { + let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); + invariant(siblingBundle !== 'root'); + let legacySiblingBundle = nullthrows( + idealBundleToLegacyBundle.get(siblingBundle), + ); + bundleGraph.createBundleReference(legacyEntryBundle, legacySiblingBundle); + } + }); + + // Step References: Add references to all bundles + for (let [asset, references] of idealGraph.assetReference) { + for (let [dependency, bundle] of references) { + let legacyBundle = nullthrows(idealBundleToLegacyBundle.get(bundle)); + bundleGraph.createAssetReference(dependency, asset, legacyBundle); + } + } + for (let {from, to} of idealBundleGraph.getAllEdges()) { + let sourceBundle = nullthrows(idealBundleGraph.getNode(from)); + if (sourceBundle === 'root') { + continue; + } + invariant(sourceBundle !== 'root'); + + let legacySourceBundle = nullthrows( + idealBundleToLegacyBundle.get(sourceBundle), + ); + + let targetBundle = nullthrows(idealBundleGraph.getNode(to)); + if (targetBundle === 'root') { + continue; + } + invariant(targetBundle !== 'root'); + let legacyTargetBundle = nullthrows( + idealBundleToLegacyBundle.get(targetBundle), + ); + bundleGraph.createBundleReference(legacySourceBundle, legacyTargetBundle); + } +} + +function createIdealGraph( + assetGraph: MutableBundleGraph, + config: ResolvedBundlerConfig, + entries: Map, +): IdealGraph { + // Asset to the bundle and group it's an entry of + let bundleRoots: Map = new Map(); + let bundles: Map = new Map(); + let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); + let assetReference: DefaultMap< + Asset, + Array<[Dependency, Bundle]>, + > = new DefaultMap(() => []); + + // A Graph of Bundles and a root node (dummy string), which models only Bundles, and connections to their + // referencing Bundle. There are no actual BundleGroup nodes, just bundles that take on that role. + let bundleGraph: Graph = new Graph(); + let stack: Array<[BundleRoot, NodeId]> = []; + + let bundleRootEdgeTypes = { + parallel: 1, + lazy: 2, + }; + // ContentGraph that models bundleRoots, with parallel & async deps only to inform reachability + let bundleRootGraph: ContentGraph< + BundleRoot | 'root', + $Values, + > = new ContentGraph(); + + let bundleGroupBundleIds: Set = new Set(); + + // Models bundleRoots and the assets that require it synchronously + let reachableRoots: ContentGraph = new ContentGraph(); + + let rootNodeId = nullthrows(bundleRootGraph.addNode('root')); + let bundleGraphRootNodeId = nullthrows(bundleGraph.addNode('root')); + bundleRootGraph.setRootNodeId(rootNodeId); + bundleGraph.setRootNodeId(bundleGraphRootNodeId); + // Step Create Entry Bundles + for (let [asset, dependency] of entries) { + let bundle = createBundle({ + asset, + target: nullthrows(dependency.target), + needsStableName: dependency.isEntry, + }); + let nodeId = bundleGraph.addNode(bundle); + bundles.set(asset.id, nodeId); + bundleRoots.set(asset, [nodeId, nodeId]); + bundleRootGraph.addEdge( + rootNodeId, + bundleRootGraph.addNodeByContentKey(asset.id, asset), + ); + bundleGraph.addEdge(bundleGraphRootNodeId, nodeId); + + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(nodeId), { + value: bundle, + type: 'bundle', + }), + dependencyPriorityEdges[dependency.priority], + ); + bundleGroupBundleIds.add(nodeId); + } + + let assets = []; + + let typeChangeIds = new Set(); + /** + * Step Create Bundles: Traverse the assetGraph (aka MutableBundleGraph) and create bundles + * for asset type changes, parallel, inline, and async or lazy dependencies, + * adding only that asset to each bundle, not its entire subgraph. + */ + assetGraph.traverse({ + enter(node, context, actions) { + if (node.type === 'asset') { + if ( + context?.type === 'dependency' && + context?.value.isEntry && + !entries.has(node.value) + ) { + // Skip whole subtrees of other targets by skipping those entries + actions.skipChildren(); + return node; + } + assets.push(node.value); + + let bundleIdTuple = bundleRoots.get(node.value); + if (bundleIdTuple && bundleIdTuple[0] === bundleIdTuple[1]) { + // Push to the stack (only) when a new bundle is created + stack.push([node.value, bundleIdTuple[0]]); + } else if (bundleIdTuple) { + // Otherwise, push on the last bundle that marks the start of a BundleGroup + stack.push([node.value, stack[stack.length - 1][1]]); + } + } else if (node.type === 'dependency') { + if (context == null) { + return node; + } let dependency = node.value; - if (bundleGraph.isDependencySkipped(dependency)) { + + if (assetGraph.isDependencySkipped(dependency)) { actions.skipChildren(); - return; + return node; } - let assets = bundleGraph.getDependencyAssets(dependency); - let resolution = bundleGraph.getResolvedAsset(dependency); - let bundleGroup = context?.bundleGroup; - // Create a new bundle for entries, lazy/parallel dependencies, isolated/inline assets. - if ( - resolution && - (!bundleGroup || - dependency.priority === 'lazy' || - dependency.priority === 'parallel' || - resolution.bundleBehavior === 'isolated' || - resolution.bundleBehavior === 'inline') - ) { - let bundleByType: Map = - context?.bundleByType ?? new Map(); + invariant(context?.type === 'asset'); + let parentAsset = context.value; - // Only create a new bundle group for entries, lazy dependencies, and isolated assets. - // Otherwise, the bundle is loaded together with the parent bundle. + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return node; + } + + for (let childAsset of assets) { if ( - !bundleGroup || dependency.priority === 'lazy' || - resolution.bundleBehavior === 'isolated' + childAsset.bundleBehavior === 'isolated' // An isolated Dependency, or Bundle must contain all assets it needs to load. ) { - bundleGroup = bundleGraph.createBundleGroup( - dependency, - nullthrows(dependency.target ?? context?.bundleGroup?.target), - ); + let bundleId = bundles.get(childAsset.id); + let bundle; + if (bundleId == null) { + let firstBundleGroup = nullthrows( + bundleGraph.getNode(stack[0][1]), + ); + invariant(firstBundleGroup !== 'root'); + bundle = createBundle({ + asset: childAsset, + target: firstBundleGroup.target, + needsStableName: + dependency.bundleBehavior === 'inline' || + childAsset.bundleBehavior === 'inline' + ? false + : dependency.isEntry || dependency.needsStableName, + bundleBehavior: + dependency.bundleBehavior ?? childAsset.bundleBehavior, + }); + bundleId = bundleGraph.addNode(bundle); + bundles.set(childAsset.id, bundleId); + bundleRoots.set(childAsset, [bundleId, bundleId]); + bundleGroupBundleIds.add(bundleId); + bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); - bundleByType = new Map(); + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } + } + + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { + value: dependency, + type: 'dependency', + }), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + dependencyPriorityEdges[dependency.priority], + ); + continue; } + if ( + parentAsset.type !== childAsset.type || + dependency.priority === 'parallel' || + childAsset.bundleBehavior === 'inline' + ) { + // The referencing bundleRoot is the root of a Bundle that first brings in another bundle (essentially the FIRST parent of a bundle, this may or may not be a bundleGroup) + let [referencingBundleRoot, bundleGroupNodeId] = nullthrows( + stack[stack.length - 1], + ); + let bundleGroup = nullthrows( + bundleGraph.getNode(bundleGroupNodeId), + ); + invariant(bundleGroup !== 'root'); - for (let asset of assets) { - let bundle = bundleGraph.createBundle({ - entryAsset: asset, - needsStableName: - dependency.bundleBehavior === 'inline' || - asset.bundleBehavior === 'inline' - ? false - : dependency.isEntry || dependency.needsStableName, - bundleBehavior: dependency.bundleBehavior ?? asset.bundleBehavior, - target: bundleGroup.target, - }); - bundleByType.set(bundle.type, bundle); - bundlesByEntryAsset.set(asset, bundle); - bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); - - // The bundle may have already been created, and the graph gave us back the original one... - if (!bundleRoots.has(bundle)) { - bundleRoots.set(bundle, [asset]); + let bundleId; + let referencingBundleId = nullthrows( + bundleRoots.get(referencingBundleRoot), + )[0]; + let referencingBundle = nullthrows( + bundleGraph.getNode(referencingBundleId), + ); + invariant(referencingBundle !== 'root'); + let bundle; + bundleId = bundles.get(childAsset.id); + + /** + * If this is an entry bundlegroup, we only allow one bundle per type in those groups + * So attempt to add the asset to the entry bundle if it's of the same type. + * This asset will be created by other dependency if it's in another bundlegroup + * and bundles of other types should be merged in the next step + */ + let bundleGroupRootAsset = nullthrows(bundleGroup.mainEntryAsset); + if ( + entries.has(bundleGroupRootAsset) && + canMerge(bundleGroupRootAsset, childAsset) && + dependency.bundleBehavior == null + ) { + bundleId = bundleGroupNodeId; } + if (bundleId == null) { + bundle = createBundle({ + // Bundles created from type changes shouldn't have an entry asset. + asset: childAsset, + type: childAsset.type, + env: childAsset.env, + bundleBehavior: + dependency.bundleBehavior ?? childAsset.bundleBehavior, + target: referencingBundle.target, + needsStableName: + childAsset.bundleBehavior === 'inline' || + dependency.bundleBehavior === 'inline' || + (dependency.priority === 'parallel' && + !dependency.needsStableName) + ? false + : referencingBundle.needsStableName, + }); + bundleId = bundleGraph.addNode(bundle); + + // Store Type-Change bundles for later since we need to know ALL bundlegroups they are part of to reduce/combine them + if (parentAsset.type !== childAsset.type) { + typeChangeIds.add(bundleId); + } + } else { + bundle = bundleGraph.getNode(bundleId); + invariant(bundle != null && bundle !== 'root'); - // If the bundle is in the same bundle group as the parent, create an asset reference - // between the dependency, the asset, and the target bundle. - if (bundleGroup === context?.bundleGroup) { - bundleGraph.createAssetReference(dependency, asset, bundle); + if ( + // If this dependency requests isolated, but the bundle is not, + // make the bundle isolated for all uses. + dependency.bundleBehavior === 'isolated' && + bundle.bundleBehavior == null + ) { + bundle.bundleBehavior = dependency.bundleBehavior; + } } - } - return { - bundleGroup, - bundleByType, - parentNode: node, - parentBundle: context?.parentBundle, - }; - } + bundles.set(childAsset.id, bundleId); + + // A bundle can belong to multiple bundlegroups, all the bundle groups of it's + // ancestors, and all async and entry bundles before it are "bundle groups" + // TODO: We may need to track bundles to all bundleGroups it belongs to in the future. + bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); + bundleGraph.addEdge(referencingBundleId, bundleId); - invariant(context != null); - invariant(context.parentNode.type === 'asset'); - invariant(context.parentBundle != null); - invariant(bundleGroup != null); - let parentAsset = context.parentNode.value; - let parentBundle = context.parentBundle; - let bundleByType = nullthrows(context.bundleByType); + if (bundleId != bundleGroupNodeId) { + dependencyBundleGraph.addEdge( + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + dependency.id, + { + value: dependency, + type: 'dependency', + }, + ), + dependencyBundleGraph.addNodeByContentKeyIfNeeded( + String(bundleId), + { + value: bundle, + type: 'bundle', + }, + ), + dependencyPriorityEdges.parallel, + ); + } - for (let asset of assets) { - if (parentAsset.type === asset.type) { + assetReference.get(childAsset).push([dependency, bundle]); continue; } + } + } + return node; + }, + exit(node) { + if (stack[stack.length - 1]?.[0] === node.value) { + stack.pop(); + } + }, + }); - let existingBundle = bundleByType.get(asset.type); - if (existingBundle) { - // If a bundle of this type has already been created in this group, - // merge this subgraph into it. - assetsToAddOnExit.get(node.value).push([existingBundle, asset]); - bundlesByEntryAsset.set(asset, existingBundle); - bundleGraph.createAssetReference(dependency, asset, existingBundle); - } else { - let bundle = bundleGraph.createBundle({ - uniqueKey: asset.id, - env: asset.env, - type: asset.type, - target: bundleGroup.target, - needsStableName: - asset.bundleBehavior === 'inline' || - dependency.bundleBehavior === 'inline' || - (dependency.priority === 'parallel' && - !dependency.needsStableName) - ? false - : parentBundle.needsStableName, - bundleBehavior: dependency.bundleBehavior ?? asset.bundleBehavior, - isSplittable: asset.isBundleSplittable ?? true, - pipeline: asset.pipeline, - }); - bundleByType.set(bundle.type, bundle); - bundlesByEntryAsset.set(asset, bundle); - bundleGraph.createAssetReference(dependency, asset, bundle); - - // Queue the asset to be added on exit of this node, so we add dependencies first. - assetsToAddOnExit.get(node.value).push([bundle, asset]); + // Step Merge Type Change Bundles: Clean up type change bundles within the exact same bundlegroups + for (let [nodeIdA, a] of bundleGraph.nodes) { + //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes + if (!typeChangeIds.has(nodeIdA) || a === 'root') continue; + let bundleABundleGroups = getBundleGroupsForBundle(nodeIdA); + for (let [nodeIdB, b] of bundleGraph.nodes) { + if ( + a !== 'root' && + b !== 'root' && + a !== b && + typeChangeIds.has(nodeIdB) && + canMerge(a, b) + ) { + let bundleBbundleGroups = getBundleGroupsForBundle(nodeIdB); + if (setEqual(bundleBbundleGroups, bundleABundleGroups)) { + let shouldMerge = true; + for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(nodeIdB)), + ALL_EDGE_TYPES, + )) { + let depNode = dependencyBundleGraph.getNode(depId); + // Cannot merge Dependency URL specifier type + if ( + depNode && + depNode.type === 'dependency' && + depNode.value.specifierType === 'url' + ) { + shouldMerge = false; + continue; + } } + if (!shouldMerge) continue; + mergeBundle(nodeIdA, nodeIdB); } + } + } + } - return { - ...context, - parentNode: node, - }; - }, - exit: node => { - if (node.type === 'dependency' && assetsToAddOnExit.has(node.value)) { - let assetsToAdd = assetsToAddOnExit.get(node.value); - for (let [bundle, asset] of assetsToAdd) { - let root = bundleRoots.get(bundle); - if (root) { - root.push(asset); - } else { - bundleRoots.set(bundle, [asset]); + /** + * Step Determine Reachability: Determine reachability for every asset from each bundleRoot. + * This is later used to determine which bundles to place each asset in. We build up two + * structures, one traversal each. ReachableRoots to store sync relationships, + * and bundleRootGraph to store the minimal availability through `parallel` and `async` relationships. + * The two graphs, are used to build up ancestorAssets, a structure which holds all availability by + * all means for each asset. + */ + for (let [root] of bundleRoots) { + if (!entries.has(root)) { + bundleRootGraph.addNodeByContentKey(root.id, root); // Add in all bundleRoots to BundleRootGraph + } + } + // ReachableRoots is a Graph of Asset Nodes which represents a BundleRoot, to all assets (non-bundleroot assets + // available to it synchronously (directly) built by traversing the assetgraph once. + for (let [root] of bundleRoots) { + // Add sync relationships to ReachableRoots + let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); + assetGraph.traverse((node, _, actions) => { + if (node.value === root) { + return; + } + if (node.type === 'dependency') { + let dependency = node.value; + + if (dependencyBundleGraph.hasContentKey(dependency.id)) { + if (dependency.priority !== 'sync') { + let assets = assetGraph.getDependencyAssets(dependency); + if (assets.length === 0) { + return; + } + invariant(assets.length === 1); + let bundleRoot = assets[0]; + let bundle = nullthrows( + bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), + ); + if ( + bundle !== 'root' && + bundle.bundleBehavior == null && + !bundle.env.isIsolated() && + bundle.env.context === root.env.context + ) { + bundleRootGraph.addEdge( + bundleRootGraph.getNodeIdByContentKey(root.id), + bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + dependency.priority === 'parallel' + ? bundleRootEdgeTypes.parallel + : bundleRootEdgeTypes.lazy, + ); } } + } - assetsToAddOnExit.delete(node.value); + if (dependency.priority !== 'sync') { + actions.skipChildren(); + } + return; + } + //asset node type + let asset = node.value; + if (asset.bundleBehavior != null || root.type !== asset.type) { + actions.skipChildren(); + return; + } + let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( + node.value.id, + node.value, + ); + reachableRoots.addEdge(rootNodeId, nodeId); + }, root); + } + // Maps a given bundleRoot to the assets reachable from it, + // and the bundleRoots reachable from each of these assets + let ancestorAssets: Map> = new Map(); + + for (let entry of entries.keys()) { + // Initialize an empty set of ancestors available to entries + ancestorAssets.set(entry, new Set()); + } + + // Step Determine Availability + // Visit nodes in a topological order, visiting parent nodes before child nodes. + + // This allows us to construct an understanding of which assets will already be + // loaded and available when a bundle runs, by pushing available assets downwards and + // computing the intersection of assets available through all possible paths to a bundle. + // We call this structure ancestorAssets, a Map that tracks a bundleRoot, + // to all assets available to it (meaning they will exist guaranteed when the bundleRoot is loaded) + // The topological sort ensures all parents are visited before the node we want to process. + for (let nodeId of bundleRootGraph.topoSort(ALL_EDGE_TYPES)) { + const bundleRoot = bundleRootGraph.getNode(nodeId); + if (bundleRoot === 'root') continue; + invariant(bundleRoot != null); + let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; + + // At a BundleRoot, we access it's available assets (via ancestorAssets), + // and add to that all assets within the bundles in that BundleGroup. + + // This set is available to all bundles in a particular bundleGroup because + // bundleGroups are just bundles loaded at the same time. However it is + // not true that a bundle's available assets = all assets of all the bundleGroups + // it belongs to. It's the intersection of those sets. + let available; + if (bundleRoot.bundleBehavior === 'isolated') { + available = new Set(); + } else { + available = new Set(ancestorAssets.get(bundleRoot)); + for (let bundleIdInGroup of [ + bundleGroupId, + ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), + ]) { + let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); + invariant(bundleInGroup !== 'root'); + if (bundleInGroup.bundleBehavior != null) { + continue; } - }, - }); - for (let [bundle, rootAssets] of bundleRoots) { - for (let asset of rootAssets) { - bundleGraph.addEntryToBundle(asset, bundle); + for (let bundleRoot of bundleInGroup.assets) { + // Assets directly connected to current bundleRoot + let assetsFromBundleRoot = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); + + for (let asset of [bundleRoot, ...assetsFromBundleRoot]) { + available.add(asset); + } + } } } - // If there's only one bundle, we can skip the rest of the steps. - skipOptimize = bundleRoots.size === 1; - if (skipOptimize) { - return; - } + // Now that we have bundleGroup availability, we will propagate that down to all the children + // of this bundleGroup. For a child, we also must maintain parallel availability. If it has + // parallel siblings that come before it, those, too, are available to it. Add those parallel + // available assets to the set of available assets for this child as well. + let children = bundleRootGraph.getNodeIdsConnectedFrom( + nodeId, + ALL_EDGE_TYPES, + ); + let parallelAvailability: Set = new Set(); - invariant(config != null); + for (let childId of children) { + let child = bundleRootGraph.getNode(childId); + invariant(child !== 'root' && child != null); + let bundleBehavior = getBundleFromBundleRoot(child).bundleBehavior; + if (bundleBehavior != null) { + continue; + } + let isParallel = bundleRootGraph.hasEdge( + nodeId, + childId, + bundleRootEdgeTypes.parallel, + ); - // Step 2: Remove asset graphs that begin with entries to other bundles. - bundleGraph.traverseBundles(bundle => { + // Most of the time, a child will have many parent bundleGroups, + // so the next time we peek at a child from another parent, we will + // intersect the availability built there with the previously computed + // availability. this ensures no matter which bundleGroup loads a particular bundle, + // it will only assume availability of assets it has under any circumstance + const childAvailableAssets = ancestorAssets.get(child); + let currentChildAvailable = isParallel + ? setUnion(parallelAvailability, available) + : available; + if (childAvailableAssets != null) { + setIntersect(childAvailableAssets, currentChildAvailable); + } else { + ancestorAssets.set(child, new Set(currentChildAvailable)); + } + if (isParallel) { + let assetsFromBundleRoot = reachableRoots + .getNodeIdsConnectedFrom( + reachableRoots.getNodeIdByContentKey(child.id), + ) + .map(id => nullthrows(reachableRoots.getNode(id))); + parallelAvailability = setUnion( + parallelAvailability, + assetsFromBundleRoot, + ); + parallelAvailability.add(child); //The next sibling should have older sibling available via parallel + } + } + } + // Step Internalize async bundles - internalize Async bundles if and only if, + // the bundle is synchronously available elsewhere. + // We can query sync assets available via reachableRoots. If the parent has + // the bundleRoot by reachableRoots AND ancestorAssets, internalize it. + for (let [id, bundleRoot] of bundleRootGraph.nodes) { + if (bundleRoot === 'root') continue; + let parentRoots = bundleRootGraph + .getNodeIdsConnectedTo(id, ALL_EDGE_TYPES) + .map(id => nullthrows(bundleRootGraph.getNode(id))); + let canDelete = + getBundleFromBundleRoot(bundleRoot).bundleBehavior !== 'isolated'; + if (parentRoots.length === 0) continue; + for (let parent of parentRoots) { + if (parent === 'root') { + canDelete = false; + continue; + } if ( - bundle.bundleBehavior === 'inline' || - bundle.bundleBehavior === 'isolated' || - !bundle.isSplittable || - bundle.env.isIsolated() + reachableRoots.hasEdge( + reachableRoots.getNodeIdByContentKey(parent.id), + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + ) || + ancestorAssets.get(parent)?.has(bundleRoot) ) { - return; + let parentBundle = bundleGraph.getNode( + nullthrows(bundles.get(parent.id)), + ); + invariant(parentBundle != null && parentBundle !== 'root'); + parentBundle.internalizedAssetIds.push(bundleRoot.id); + } else { + canDelete = false; } + } + if (canDelete) { + deleteBundle(bundleRoot); + } + } + // Step Insert Or Share: Place all assets into bundles or create shared bundles. Each asset + // is placed into a single bundle based on the bundle entries it is reachable from. + // This creates a maximally code split bundle graph with no duplication. + for (let asset of assets) { + // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means + let reachable: Array = getReachableBundleRoots( + asset, + reachableRoots, + ).reverse(); + + let reachableEntries = []; + let reachableNonEntries = []; - // Skip bundles where the entry is reachable in a parent bundle. This can occur when both synchronously and - // asynchronously importing an asset from a bundle. This asset will later be internalized into the parent. - let entries = bundle.getEntryAssets(); - let mainEntry = entries[0]; + // Filter out entries, since they can't have shared bundles. + // Neither can non-splittable, isolated, or needing of stable name bundles. + // Reserve those filtered out bundles since we add the asset back into them. + for (let a of reachable) { if ( - mainEntry == null || - entries.length !== 1 || - bundleGraph.isAssetReachableFromBundle(mainEntry, bundle) + entries.has(a) || + !a.isBundleSplittable || + getBundleFromBundleRoot(a).needsStableName || + getBundleFromBundleRoot(a).bundleBehavior === 'isolated' ) { - return; + reachableEntries.push(a); + } else { + reachableNonEntries.push(a); } + } + reachable = reachableNonEntries; + + // Filter out bundles from this asset's reachable array if + // bundle does not contain the asset in its ancestry + reachable = reachable.filter(b => !ancestorAssets.get(b)?.has(asset)); - let candidates = bundleGraph.getBundlesWithAsset(mainEntry).filter( - containingBundle => - containingBundle.id !== bundle.id && - // Don't add to BundleGroups for entry bundles, as that would require - // another entry bundle depending on these conditions, making it difficult - // to predict and reference. - // TODO: reconsider this. This is only true for the global output format. - !containingBundle.needsStableName && - containingBundle.bundleBehavior !== 'inline' && - containingBundle.bundleBehavior !== 'isolated' && - containingBundle.isSplittable, + // Finally, filter out bundleRoots (bundles) from this assets + // reachable if they are subgraphs, and reuse that subgraph bundle + // by drawing an edge. Essentially, if two bundles within an asset's + // reachable array, have an ancestor-subgraph relationship, draw that edge. + // This allows for us to reuse a bundle instead of making a shared bundle if + // a bundle represents the exact set of assets a set of bundles would share + + // if a bundle b is a subgraph of another bundle f, reuse it, drawing an edge between the two + let canReuse: Set = new Set(); + for (let candidateSourceBundleRoot of reachable) { + let candidateSourceBundleId = nullthrows( + bundles.get(candidateSourceBundleRoot.id), ); + if (candidateSourceBundleRoot.env.isIsolated()) { + continue; + } + let reuseableBundleId = bundles.get(asset.id); + if (reuseableBundleId != null) { + canReuse.add(candidateSourceBundleRoot); + bundleGraph.addEdge(candidateSourceBundleId, reuseableBundleId); - for (let candidate of candidates) { - let bundleGroups = - bundleGraph.getBundleGroupsContainingBundle(candidate); - if ( - Array.from(bundleGroups).every( - group => - bundleGraph.getBundlesInBundleGroup(group).length < - config.maxParallelRequests, - ) - ) { - bundleGraph.createBundleReference(candidate, bundle); - bundleGraph.removeAssetGraphFromBundle(mainEntry, candidate); + let reusableBundle = bundleGraph.getNode(reuseableBundleId); + invariant(reusableBundle !== 'root' && reusableBundle != null); + reusableBundle.sourceBundles.add(candidateSourceBundleId); + } else { + // Asset is not a bundleRoot, but if its ancestor bundle (in the asset's reachable) can be + // reused as a subgraph of another bundleRoot in its reachable, reuse it + for (let otherReuseCandidate of reachable) { + if (candidateSourceBundleRoot === otherReuseCandidate) continue; + let reusableCandidateReachable = getReachableBundleRoots( + otherReuseCandidate, + reachableRoots, + ).filter(b => !ancestorAssets.get(b)?.has(otherReuseCandidate)); + if (reusableCandidateReachable.includes(candidateSourceBundleRoot)) { + let reusableBundleId = nullthrows( + bundles.get(otherReuseCandidate.id), + ); + canReuse.add(candidateSourceBundleRoot); + bundleGraph.addEdge( + nullthrows(bundles.get(candidateSourceBundleRoot.id)), + reusableBundleId, + ); + let reusableBundle = bundleGraph.getNode(reusableBundleId); + invariant(reusableBundle !== 'root' && reusableBundle != null); + reusableBundle.sourceBundles.add(candidateSourceBundleId); + } } } - }); - - // Step 3: Remove assets that are duplicated in a parent bundle. - deduplicate(bundleGraph); - internalizeReachableAsyncDependencies(bundleGraph); - }, - optimize({bundleGraph, config}) { - // if only one bundle, no need to optimize - if (skipOptimize) { - return; } + //Bundles that are reused should not be considered for shared bundles, so filter them out + reachable = reachable.filter(b => !canReuse.has(b)); - invariant(config != null); - - // Step 5: Find duplicated assets in different bundle groups, and separate them into their own parallel bundles. - // If multiple assets are always seen together in the same bundles, combine them together. - // If the sub-graph from an asset is >= 30kb, and the number of parallel requests in the bundle group is < 5, create a new bundle containing the sub-graph. - let candidateBundles: Map< - string, - {| - assets: Array, - sourceBundles: Set, - size: number, - |}, - > = new Map(); - - bundleGraph.traverse((node, ctx, actions) => { - if (node.type !== 'asset') { - return; - } + // Add assets to non-splittable bundles. + for (let entry of reachableEntries) { + let entryBundleId = nullthrows(bundles.get(entry.id)); + let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); + invariant(entryBundle !== 'root'); + entryBundle.assets.add(asset); + entryBundle.size += asset.stats.size; + } - let asset = node.value; - let containingBundles = bundleGraph - .getBundlesWithAsset(asset) - // Don't create shared bundles from entry bundles, as that would require - // another entry bundle depending on these conditions, making it difficult - // to predict and reference. - // TODO: reconsider this. This is only true for the global output format. - // This also currently affects other bundles with stable names, e.g. service workers. - .filter(b => { - let entries = b.getEntryAssets(); - - return ( - !b.needsStableName && - b.isSplittable && - entries.every(entry => entry.id !== asset.id) - ); + // Create shared bundles for splittable bundles. + if (reachable.length > config.minBundles) { + let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); + let key = reachable.map(a => a.id).join(','); + let bundleId = bundles.get(key); + let bundle; + if (bundleId == null) { + let firstSourceBundle = nullthrows( + bundleGraph.getNode(sourceBundles[0]), + ); + invariant(firstSourceBundle !== 'root'); + bundle = createBundle({ + target: firstSourceBundle.target, + type: firstSourceBundle.type, + env: firstSourceBundle.env, }); + bundle.sourceBundles = new Set(sourceBundles); + let sharedInternalizedAssets = new Set( + firstSourceBundle.internalizedAssetIds, + ); - if (containingBundles.length > config.minBundles) { - let id = containingBundles - .map(b => b.id) - .sort() - .join(':'); - - let candidate = candidateBundles.get(id); - if (candidate) { - candidate.assets.push(asset); - for (let bundle of containingBundles) { - candidate.sourceBundles.add(bundle); - } - candidate.size += bundleGraph.getTotalSize(asset); - } else { - candidateBundles.set(id, { - assets: [asset], - sourceBundles: new Set(containingBundles), - size: bundleGraph.getTotalSize(asset), - }); + for (let p of sourceBundles) { + let parentBundle = nullthrows(bundleGraph.getNode(p)); + invariant(parentBundle !== 'root'); + if (parentBundle === firstSourceBundle) continue; + setIntersect( + sharedInternalizedAssets, + new Set(parentBundle.internalizedAssetIds), + ); } - - // Skip children from consideration since we added a parent already. - actions.skipChildren(); + bundle.internalizedAssetIds = [...sharedInternalizedAssets]; + bundleId = bundleGraph.addNode(bundle); + bundles.set(key, bundleId); + } else { + bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); } - }); + bundle.assets.add(asset); + bundle.size += asset.stats.size; - // Sort candidates by size (consider larger bundles first), and ensure they meet the size threshold - let sortedCandidates: Array<{| - assets: Array, - sourceBundles: Set, - size: number, - |}> = Array.from(candidateBundles.values()) - .filter(bundle => bundle.size >= config.minBundleSize) - .sort((a, b) => b.size - a.size); - - for (let {assets, sourceBundles} of sortedCandidates) { - let eligibleSourceBundles = new Set(); - - for (let bundle of sourceBundles) { - // Find all bundle groups connected to the original bundles - let bundleGroups = bundleGraph.getBundleGroupsContainingBundle(bundle); - // Check if all bundle groups are within the parallel request limit - if ( - bundleGroups.every( - group => - bundleGraph.getBundlesInBundleGroup(group).length < - config.maxParallelRequests, - ) - ) { - eligibleSourceBundles.add(bundle); + for (let sourceBundleId of sourceBundles) { + if (bundleId !== sourceBundleId) { + bundleGraph.addEdge(sourceBundleId, bundleId); } } - // Do not create a shared bundle unless there are at least 2 source bundles - if (eligibleSourceBundles.size < 2) { - continue; + dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { + value: bundle, + type: 'bundle', + }); + } else if (reachable.length <= config.minBundles) { + for (let root of reachable) { + let bundle = nullthrows( + bundleGraph.getNode(nullthrows(bundles.get(root.id))), + ); + invariant(bundle !== 'root'); + bundle.assets.add(asset); + bundle.size += asset.stats.size; } + } + } + // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into + // their source bundles, and remove the bundle. + // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained + for (let [bundleNodeId, bundle] of bundleGraph.nodes) { + if (bundle === 'root') continue; + if ( + bundle.sourceBundles.size > 0 && + bundle.mainEntryAsset == null && + bundle.size < config.minBundleSize + ) { + removeBundle(bundleGraph, bundleNodeId, assetReference); + } + } - let [firstBundle] = [...eligibleSourceBundles]; - let sharedBundle = bundleGraph.createBundle({ - uniqueKey: hashString( - [...eligibleSourceBundles].map(b => b.id).join(':'), - ), - // Allow this bundle to be deduplicated. It shouldn't be further split. - // TODO: Reconsider bundle/asset flags. - isSplittable: true, - env: firstBundle.env, - target: firstBundle.target, - type: firstBundle.type, + // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit. + for (let bundleGroupId of bundleGraph.getNodeIdsConnectedFrom(rootNodeId)) { + // Find shared bundles in this bundle group. + let bundleId = bundleGroupId; + + // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained + let bundleIdsInGroup = getBundlesForBundleGroup(bundleId); //get all bundlegrups this bundle is an ancestor of + if (bundleIdsInGroup.length > config.maxParallelRequests) { + let sharedBundleIdsInBundleGroup = bundleIdsInGroup.filter(b => { + let bundle = nullthrows(bundleGraph.getNode(b)); + // shared bundles must have source bundles, we could have a bundle + // connected to another bundle that isnt a shared bundle, so check + return ( + bundle !== 'root' && bundle.sourceBundles.size > 0 && bundleId != b + ); }); - // Remove all of the root assets from each of the original bundles - // and reference the new shared bundle. - for (let asset of assets) { - bundleGraph.addAssetGraphToBundle(asset, sharedBundle); + let numBundlesInGroup = bundleIdsInGroup.length; + // Sort the bundles so the smallest ones are removed first. + let sharedBundlesInGroup = sharedBundleIdsInBundleGroup + .map(id => ({ + id, + bundle: nullthrows(bundleGraph.getNode(id)), + })) + .map(({id, bundle}) => { + // For Flow + invariant(bundle !== 'root'); + return {id, bundle}; + }) + .sort((a, b) => b.bundle.size - a.bundle.size); + + // Remove bundles until the bundle group is within the parallel request limit. + while ( + sharedBundlesInGroup.length > 0 && + numBundlesInGroup > config.maxParallelRequests + ) { + let bundleTuple = sharedBundlesInGroup.pop(); + let bundleToRemove = bundleTuple.bundle; + let bundleIdToRemove = bundleTuple.id; + //TODO add integration test where bundles in bunlde group > max parallel request limit & only remove a couple shared bundles + // but total # bundles still exceeds limit due to non shared bundles + + // Add all assets in the shared bundle into the source bundles that are within this bundle group. + let sourceBundles = [...bundleToRemove.sourceBundles].filter(b => + bundleIdsInGroup.includes(b), + ); - for (let bundle of eligibleSourceBundles) { - { - bundleGraph.createBundleReference(bundle, sharedBundle); - bundleGraph.removeAssetGraphFromBundle(asset, bundle); + for (let sourceBundleId of sourceBundles) { + let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); + invariant(sourceBundle !== 'root'); + bundleToRemove.sourceBundles.delete(sourceBundleId); + for (let asset of bundleToRemove.assets) { + sourceBundle.assets.add(asset); + sourceBundle.size += asset.stats.size; + } + //This case is specific to reused bundles, which can have shared bundles attached to it + for (let childId of bundleGraph.getNodeIdsConnectedFrom( + bundleIdToRemove, + )) { + let child = bundleGraph.getNode(childId); + invariant(child !== 'root' && child != null); + child.sourceBundles.add(sourceBundleId); + bundleGraph.addEdge(sourceBundleId, childId); + } + // needs to add test case where shared bundle is removed from ONE bundlegroup but not from the whole graph! + // Remove the edge from this bundle group to the shared bundle. + // If there is now only a single bundle group that contains this bundle, + // merge it into the remaining source bundles. If it is orphaned entirely, remove it. + let incomingNodeCount = + bundleGraph.getNodeIdsConnectedTo(bundleIdToRemove).length; + + if ( + incomingNodeCount <= 2 && + //Never fully remove reused bundles + bundleToRemove.mainEntryAsset == null + ) { + // If one bundle group removes a shared bundle, but the other *can* keep it, still remove because that shared bundle is pointless (only one source bundle) + removeBundle(bundleGraph, bundleIdToRemove, assetReference); + // Stop iterating through bundleToRemove's sourceBundles as the bundle has been removed. + break; + } else { + bundleGraph.removeEdge(sourceBundleId, bundleIdToRemove); } } + numBundlesInGroup--; } } + } + function deleteBundle(bundleRoot: BundleRoot) { + bundleGraph.removeNode(nullthrows(bundles.get(bundleRoot.id))); + bundleRoots.delete(bundleRoot); + bundles.delete(bundleRoot.id); + if (reachableRoots.hasContentKey(bundleRoot.id)) { + reachableRoots.replaceNodeIdsConnectedTo( + reachableRoots.getNodeIdByContentKey(bundleRoot.id), + [], + ); + } + if (bundleRootGraph.hasContentKey(bundleRoot.id)) { + bundleRootGraph.removeNode( + bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), + ); + } + } + function getBundleGroupsForBundle(nodeId: NodeId) { + let bundleGroupBundleIds = new Set(); + bundleGraph.traverseAncestors(nodeId, ancestorId => { + if ( + bundleGraph + .getNodeIdsConnectedTo(ancestorId) //if node is root, then dont add, otherwise do add. + .includes(bundleGraph.rootNodeId) + ) { + bundleGroupBundleIds.add(ancestorId); + } + }); + return bundleGroupBundleIds; + } + function getBundlesForBundleGroup(bundleGroupId) { + let bundlesInABundleGroup = []; + bundleGraph.traverse(nodeId => { + bundlesInABundleGroup.push(nodeId); + }, bundleGroupId); + return bundlesInABundleGroup; + } - // Remove assets that are duplicated between shared bundles. - deduplicate(bundleGraph); - internalizeReachableAsyncDependencies(bundleGraph); - }, -}): Bundler); + function mergeBundle(mainNodeId: NodeId, otherNodeId: NodeId) { + //merges assets of "otherRoot" into "mainBundleRoot" + let a = nullthrows(bundleGraph.getNode(mainNodeId)); + let b = nullthrows(bundleGraph.getNode(otherNodeId)); + invariant(a !== 'root' && b !== 'root'); + let bundleRootB = nullthrows(b.mainEntryAsset); + let mainBundleRoot = nullthrows(a.mainEntryAsset); + for (let asset of a.assets) { + b.assets.add(asset); + } + a.assets = b.assets; + for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( + dependencyBundleGraph.getNodeIdByContentKey(String(otherNodeId)), + ALL_EDGE_TYPES, + )) { + dependencyBundleGraph.replaceNodeIdsConnectedTo(depId, [ + dependencyBundleGraph.getNodeIdByContentKey(String(mainNodeId)), + ]); + } -function deduplicate(bundleGraph: MutableBundleGraph) { - bundleGraph.traverse(node => { - if (node.type === 'asset') { - let asset = node.value; - // Search in reverse order, so bundles that are loaded keep the duplicated asset, not later ones. - // This ensures that the earlier bundle is able to execute before the later one. - let bundles = bundleGraph.getBundlesWithAsset(asset).reverse(); - for (let bundle of bundles) { - if ( - bundle.hasAsset(asset) && - bundleGraph.isAssetReachableFromBundle(asset, bundle) - ) { - bundleGraph.removeAssetGraphFromBundle(asset, bundle); - } - } + //clean up asset reference + for (let dependencyTuple of assetReference.get(bundleRootB)) { + dependencyTuple[1] = a; } - }); + //add in any lost edges + for (let nodeId of bundleGraph.getNodeIdsConnectedTo(otherNodeId)) { + bundleGraph.addEdge(nodeId, mainNodeId); + } + deleteBundle(bundleRootB); + let bundleGroupOfMain = nullthrows(bundleRoots.get(mainBundleRoot))[1]; + bundleRoots.set(bundleRootB, [mainNodeId, bundleGroupOfMain]); + bundles.set(bundleRootB.id, mainNodeId); + } + function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { + let bundle = bundleGraph.getNode( + nullthrows(bundleRoots.get(bundleRoot))[0], + ); + invariant(bundle !== 'root' && bundle != null); + return bundle; + } + + return { + bundleGraph, + dependencyBundleGraph, + bundleGroupBundleIds, + assetReference, + }; } const CONFIG_SCHEMA: SchemaEntity = { @@ -456,10 +1217,78 @@ const CONFIG_SCHEMA: SchemaEntity = { additionalProperties: false, }; -async function loadBundlerConfig(config: Config, options: PluginOptions) { +function createBundle(opts: {| + uniqueKey?: string, + target: Target, + asset?: Asset, + env?: Environment, + type?: string, + needsStableName?: boolean, + bundleBehavior?: ?BundleBehavior, +|}): Bundle { + if (opts.asset == null) { + return { + uniqueKey: opts.uniqueKey, + assets: new Set(), + internalizedAssetIds: [], + mainEntryAsset: null, + size: 0, + sourceBundles: new Set(), + target: opts.target, + type: nullthrows(opts.type), + env: nullthrows(opts.env), + needsStableName: Boolean(opts.needsStableName), + bundleBehavior: opts.bundleBehavior, + }; + } + + let asset = nullthrows(opts.asset); + return { + uniqueKey: opts.uniqueKey, + assets: new Set([asset]), + internalizedAssetIds: [], + mainEntryAsset: asset, + size: asset.stats.size, + sourceBundles: new Set(), + target: opts.target, + type: opts.type ?? asset.type, + env: opts.env ?? asset.env, + needsStableName: Boolean(opts.needsStableName), + bundleBehavior: opts.bundleBehavior ?? asset.bundleBehavior, + }; +} + +function removeBundle( + bundleGraph: Graph, + bundleId: NodeId, + assetReference: DefaultMap>, +) { + let bundle = nullthrows(bundleGraph.getNode(bundleId)); + invariant(bundle !== 'root'); + for (let asset of bundle.assets) { + assetReference.set( + asset, + assetReference.get(asset).filter(t => !t.includes(bundle)), + ); + for (let sourceBundleId of bundle.sourceBundles) { + let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); + invariant(sourceBundle !== 'root'); + sourceBundle.assets.add(asset); + sourceBundle.size += asset.stats.size; + } + } + + bundleGraph.removeNode(bundleId); +} + +async function loadBundlerConfig( + config: Config, + options: PluginOptions, +): Promise { let conf = await config.getConfig([], { packageKey: '@parcel/bundler-default', }); + if (!conf) { return HTTP_OPTIONS['2']; } @@ -489,58 +1318,45 @@ async function loadBundlerConfig(config: Config, options: PluginOptions) { }; } -function internalizeReachableAsyncDependencies( - bundleGraph: MutableBundleGraph, -): void { - // Mark async dependencies on assets that are already available in - // the bundle as internally resolvable. This removes the dependency between - // the bundle and the bundle group providing that asset. If all connections - // to that bundle group are removed, remove that bundle group. - let asyncBundleGroups: Set = new Set(); - bundleGraph.traverse((node, _, actions) => { - if ( - node.type !== 'dependency' || - node.value.isEntry || - node.value.priority !== 'lazy' - ) { - return; - } - - if (bundleGraph.isDependencySkipped(node.value)) { - actions.skipChildren(); - return; - } - - let dependency = node.value; - if (dependency.specifierType === 'url') { - // Don't internalize dependencies on URLs, e.g. `new Worker('foo.js')` - return; - } - - let resolution = bundleGraph.getResolvedAsset(dependency); - if (resolution == null) { - return; - } - - let externalResolution = bundleGraph.resolveAsyncDependency(dependency); - if (externalResolution?.type === 'bundle_group') { - asyncBundleGroups.add(externalResolution.value); - } +function getReachableBundleRoots(asset, graph): Array { + return graph + .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) + .map(nodeId => nullthrows(graph.getNode(nodeId))); +} - for (let bundle of bundleGraph.getBundlesWithDependency(dependency)) { - if ( - bundle.hasAsset(resolution) || - bundleGraph.isAssetReachableFromBundle(resolution, bundle) - ) { - bundleGraph.internalizeAsyncDependency(bundle, dependency); +function getEntryByTarget( + bundleGraph: MutableBundleGraph, +): DefaultMap> { + // Find entries from assetGraph per target + let targets: DefaultMap> = new DefaultMap( + () => new Map(), + ); + bundleGraph.traverse({ + enter(node, context, actions) { + if (node.type !== 'asset') { + return node; } - } + invariant( + context != null && + context.type === 'dependency' && + context.value.isEntry && + context.value.target != null, + ); + targets.get(context.value.target.distDir).set(node.value, context.value); + actions.skipChildren(); + return node; + }, }); + return targets; +} - // Remove any bundle groups that no longer have any parent bundles. - for (let bundleGroup of asyncBundleGroups) { - if (bundleGraph.getParentBundlesOfBundleGroup(bundleGroup).length === 0) { - bundleGraph.removeBundleGroup(bundleGroup); - } - } +function canMerge(a, b) { + // Bundles can be merged if they have the same type and environment, + // unless they are explicitly marked as isolated or inline. + return ( + a.type === b.type && + a.env.context === b.env.context && + a.bundleBehavior == null && + b.bundleBehavior == null + ); } diff --git a/packages/bundlers/experimental/package.json b/packages/bundlers/experimental/package.json deleted file mode 100644 index 2ab27f22e22..00000000000 --- a/packages/bundlers/experimental/package.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "name": "@parcel/bundler-experimental", - "version": "2.7.0", - "license": "MIT", - "publishConfig": { - "access": "public" - }, - "funding": { - "type": "opencollective", - "url": "https://opencollective.com/parcel" - }, - "repository": { - "type": "git", - "url": "https://github.com/parcel-bundler/parcel.git" - }, - "main": "lib/ExperimentalBundler.js", - "source": "src/ExperimentalBundler.js", - "engines": { - "node": ">= 12.0.0", - "parcel": "^2.7.0" - }, - "dependencies": { - "@parcel/diagnostic": "2.7.0", - "@parcel/graph": "2.7.0", - "@parcel/hash": "2.7.0", - "@parcel/plugin": "2.7.0", - "@parcel/utils": "2.7.0", - "nullthrows": "^1.1.1" - } -} diff --git a/packages/bundlers/experimental/src/ExperimentalBundler.js b/packages/bundlers/experimental/src/ExperimentalBundler.js deleted file mode 100644 index d39fc39a99c..00000000000 --- a/packages/bundlers/experimental/src/ExperimentalBundler.js +++ /dev/null @@ -1,1362 +0,0 @@ -// @flow strict-local - -import type { - Asset, - Bundle as LegacyBundle, - BundleBehavior, - BundleGroup, - Dependency, - Environment, - Config, - MutableBundleGraph, - PluginOptions, - Target, -} from '@parcel/types'; -import type {NodeId} from '@parcel/graph'; -import type {SchemaEntity} from '@parcel/utils'; -import {ContentGraph, Graph} from '@parcel/graph'; - -import invariant from 'assert'; -import {ALL_EDGE_TYPES} from '@parcel/graph'; -import {Bundler} from '@parcel/plugin'; -import { - setIntersect, - setUnion, - setEqual, - validateSchema, - DefaultMap, -} from '@parcel/utils'; -import nullthrows from 'nullthrows'; -import {encodeJSONKeyComponent} from '@parcel/diagnostic'; - -type BundlerConfig = {| - http?: number, - minBundles?: number, - minBundleSize?: number, - maxParallelRequests?: number, -|}; - -type ResolvedBundlerConfig = {| - minBundles: number, - minBundleSize: number, - maxParallelRequests: number, -|}; - -// Default options by http version. -const HTTP_OPTIONS = { - '1': { - minBundles: 1, - minBundleSize: 30000, - maxParallelRequests: 6, - }, - '2': { - minBundles: 1, - minBundleSize: 20000, - maxParallelRequests: 25, - }, -}; - -type AssetId = string; - -/* BundleRoot - An asset that is the main entry of a Bundle. */ -type BundleRoot = Asset; -export type Bundle = {| - uniqueKey: ?string, - assets: Set, - internalizedAssetIds: Array, - bundleBehavior?: ?BundleBehavior, - needsStableName: boolean, - mainEntryAsset: ?Asset, - size: number, - sourceBundles: Set, - target: Target, - env: Environment, - type: string, -|}; - -const dependencyPriorityEdges = { - sync: 1, - parallel: 2, - lazy: 3, -}; - -type DependencyBundleGraph = ContentGraph< - | {| - value: Bundle, - type: 'bundle', - |} - | {| - value: Dependency, - type: 'dependency', - |}, - number, ->; -// IdealGraph is the structure we will pass to decorate, -// which mutates the assetGraph into the bundleGraph we would -// expect from default bundler -type IdealGraph = {| - dependencyBundleGraph: DependencyBundleGraph, - bundleGraph: Graph, - bundleGroupBundleIds: Set, - assetReference: DefaultMap>, -|}; - -/** - * - * The Bundler works by creating an IdealGraph, which contains a BundleGraph that models bundles - * connected to othervbundles by what references them, and thus models BundleGroups. - * - * First, we enter `bundle({bundleGraph, config})`. Here, "bundleGraph" is actually just the - * assetGraph turned into a type `MutableBundleGraph`, which will then be mutated in decorate, - * and turned into what we expect the bundleGraph to be as per the old (default) bundler structure - * & what the rest of Parcel expects a BundleGraph to be. - * - * `bundle({bundleGraph, config})` First gets a Mapping of target to entries, In most cases there is - * only one target, and one or more entries. (Targets are pertinent in monorepos or projects where you - * will have two or more distDirs, or output folders.) Then calls create IdealGraph and Decorate per target. - * - */ -export default (new Bundler({ - loadConfig({config, options}) { - return loadBundlerConfig(config, options); - }, - - bundle({bundleGraph, config}) { - let targetMap = getEntryByTarget(bundleGraph); // Organize entries by target output folder/ distDir - let graphs = []; - for (let entries of targetMap.values()) { - // Create separate bundleGraphs per distDir - graphs.push(createIdealGraph(bundleGraph, config, entries)); - } - for (let g of graphs) { - decorateLegacyGraph(g, bundleGraph); //mutate original graph - } - }, - optimize() {}, -}): Bundler); - -function decorateLegacyGraph( - idealGraph: IdealGraph, - bundleGraph: MutableBundleGraph, -): void { - let idealBundleToLegacyBundle: Map = new Map(); - - let { - bundleGraph: idealBundleGraph, - dependencyBundleGraph, - bundleGroupBundleIds, - } = idealGraph; - let entryBundleToBundleGroup: Map = new Map(); - // Step Create Bundles: Create bundle groups, bundles, and shared bundles and add assets to them - for (let [bundleNodeId, idealBundle] of idealBundleGraph.nodes) { - if (idealBundle === 'root') continue; - let entryAsset = idealBundle.mainEntryAsset; - let bundleGroup; - let bundle; - - if (bundleGroupBundleIds.has(bundleNodeId)) { - let dependencies = dependencyBundleGraph - .getNodeIdsConnectedTo( - dependencyBundleGraph.getNodeIdByContentKey(String(bundleNodeId)), - ALL_EDGE_TYPES, - ) - .map(nodeId => { - let dependency = nullthrows(dependencyBundleGraph.getNode(nodeId)); - invariant(dependency.type === 'dependency'); - return dependency.value; - }); - for (let dependency of dependencies) { - bundleGroup = bundleGraph.createBundleGroup( - dependency, - idealBundle.target, - ); - } - invariant(bundleGroup); - entryBundleToBundleGroup.set(bundleNodeId, bundleGroup); - - bundle = nullthrows( - bundleGraph.createBundle({ - entryAsset: nullthrows(entryAsset), - needsStableName: idealBundle.needsStableName, - bundleBehavior: idealBundle.bundleBehavior, - target: idealBundle.target, - }), - ); - - bundleGraph.addBundleToBundleGroup(bundle, bundleGroup); - } else if (idealBundle.sourceBundles.size > 0) { - bundle = nullthrows( - bundleGraph.createBundle({ - uniqueKey: - [...idealBundle.assets].map(asset => asset.id).join(',') + - [...idealBundle.sourceBundles].join(','), - needsStableName: idealBundle.needsStableName, - bundleBehavior: idealBundle.bundleBehavior, - type: idealBundle.type, - target: idealBundle.target, - env: idealBundle.env, - }), - ); - } else if (idealBundle.uniqueKey != null) { - bundle = nullthrows( - bundleGraph.createBundle({ - uniqueKey: idealBundle.uniqueKey, - needsStableName: idealBundle.needsStableName, - bundleBehavior: idealBundle.bundleBehavior, - type: idealBundle.type, - target: idealBundle.target, - env: idealBundle.env, - }), - ); - } else { - invariant(entryAsset != null); - bundle = nullthrows( - bundleGraph.createBundle({ - entryAsset, - needsStableName: idealBundle.needsStableName, - bundleBehavior: idealBundle.bundleBehavior, - target: idealBundle.target, - }), - ); - } - - idealBundleToLegacyBundle.set(idealBundle, bundle); - - for (let asset of idealBundle.assets) { - bundleGraph.addAssetToBundle(asset, bundle); - } - } - // Step Internalization: Internalize dependencies for bundles - for (let [, idealBundle] of idealBundleGraph.nodes) { - if (idealBundle === 'root') continue; - let bundle = nullthrows(idealBundleToLegacyBundle.get(idealBundle)); - for (let internalized of idealBundle.internalizedAssetIds) { - let incomingDeps = bundleGraph.getIncomingDependencies( - bundleGraph.getAssetById(internalized), - ); - for (let incomingDep of incomingDeps) { - if ( - incomingDep.priority === 'lazy' && - incomingDep.specifierType !== 'url' && - bundle.hasDependency(incomingDep) - ) { - bundleGraph.internalizeAsyncDependency(bundle, incomingDep); - } - } - } - } - - // Step Add to BundleGroups: Add bundles to their bundle groups - idealBundleGraph.traverse((nodeId, _, actions) => { - let node = idealBundleGraph.getNode(nodeId); - if (node === 'root') { - return; - } - actions.skipChildren(); - - let outboundNodeIds = idealBundleGraph.getNodeIdsConnectedFrom(nodeId); - let entryBundle = nullthrows(idealBundleGraph.getNode(nodeId)); - invariant(entryBundle !== 'root'); - let legacyEntryBundle = nullthrows( - idealBundleToLegacyBundle.get(entryBundle), - ); - - for (let id of outboundNodeIds) { - let siblingBundle = nullthrows(idealBundleGraph.getNode(id)); - invariant(siblingBundle !== 'root'); - let legacySiblingBundle = nullthrows( - idealBundleToLegacyBundle.get(siblingBundle), - ); - bundleGraph.createBundleReference(legacyEntryBundle, legacySiblingBundle); - } - }); - - // Step References: Add references to all bundles - for (let [asset, references] of idealGraph.assetReference) { - for (let [dependency, bundle] of references) { - let legacyBundle = nullthrows(idealBundleToLegacyBundle.get(bundle)); - bundleGraph.createAssetReference(dependency, asset, legacyBundle); - } - } - - for (let {from, to} of idealBundleGraph.getAllEdges()) { - let sourceBundle = nullthrows(idealBundleGraph.getNode(from)); - if (sourceBundle === 'root') { - continue; - } - invariant(sourceBundle !== 'root'); - - let legacySourceBundle = nullthrows( - idealBundleToLegacyBundle.get(sourceBundle), - ); - - let targetBundle = nullthrows(idealBundleGraph.getNode(to)); - if (targetBundle === 'root') { - continue; - } - invariant(targetBundle !== 'root'); - let legacyTargetBundle = nullthrows( - idealBundleToLegacyBundle.get(targetBundle), - ); - bundleGraph.createBundleReference(legacySourceBundle, legacyTargetBundle); - } -} - -function createIdealGraph( - assetGraph: MutableBundleGraph, - config: ResolvedBundlerConfig, - entries: Map, -): IdealGraph { - // Asset to the bundle and group it's an entry of - let bundleRoots: Map = new Map(); - let bundles: Map = new Map(); - let dependencyBundleGraph: DependencyBundleGraph = new ContentGraph(); - let assetReference: DefaultMap< - Asset, - Array<[Dependency, Bundle]>, - > = new DefaultMap(() => []); - - // A Graph of Bundles and a root node (dummy string), which models only Bundles, and connections to their - // referencing Bundle. There are no actual BundleGroup nodes, just bundles that take on that role. - let bundleGraph: Graph = new Graph(); - let stack: Array<[BundleRoot, NodeId]> = []; - - let bundleRootEdgeTypes = { - parallel: 1, - lazy: 2, - }; - // ContentGraph that models bundleRoots, with parallel & async deps only to inform reachability - let bundleRootGraph: ContentGraph< - BundleRoot | 'root', - $Values, - > = new ContentGraph(); - - let bundleGroupBundleIds: Set = new Set(); - - // Models bundleRoots and the assets that require it synchronously - let reachableRoots: ContentGraph = new ContentGraph(); - - let rootNodeId = nullthrows(bundleRootGraph.addNode('root')); - let bundleGraphRootNodeId = nullthrows(bundleGraph.addNode('root')); - bundleRootGraph.setRootNodeId(rootNodeId); - bundleGraph.setRootNodeId(bundleGraphRootNodeId); - // Step Create Entry Bundles - for (let [asset, dependency] of entries) { - let bundle = createBundle({ - asset, - target: nullthrows(dependency.target), - needsStableName: dependency.isEntry, - }); - let nodeId = bundleGraph.addNode(bundle); - bundles.set(asset.id, nodeId); - bundleRoots.set(asset, [nodeId, nodeId]); - bundleRootGraph.addEdge( - rootNodeId, - bundleRootGraph.addNodeByContentKey(asset.id, asset), - ); - bundleGraph.addEdge(bundleGraphRootNodeId, nodeId); - - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { - value: dependency, - type: 'dependency', - }), - dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(nodeId), { - value: bundle, - type: 'bundle', - }), - dependencyPriorityEdges[dependency.priority], - ); - bundleGroupBundleIds.add(nodeId); - } - - let assets = []; - - let typeChangeIds = new Set(); - /** - * Step Create Bundles: Traverse the assetGraph (aka MutableBundleGraph) and create bundles - * for asset type changes, parallel, inline, and async or lazy dependencies, - * adding only that asset to each bundle, not its entire subgraph. - */ - assetGraph.traverse({ - enter(node, context, actions) { - if (node.type === 'asset') { - if ( - context?.type === 'dependency' && - context?.value.isEntry && - !entries.has(node.value) - ) { - // Skip whole subtrees of other targets by skipping those entries - actions.skipChildren(); - return node; - } - assets.push(node.value); - - let bundleIdTuple = bundleRoots.get(node.value); - if (bundleIdTuple && bundleIdTuple[0] === bundleIdTuple[1]) { - // Push to the stack (only) when a new bundle is created - stack.push([node.value, bundleIdTuple[0]]); - } else if (bundleIdTuple) { - // Otherwise, push on the last bundle that marks the start of a BundleGroup - stack.push([node.value, stack[stack.length - 1][1]]); - } - } else if (node.type === 'dependency') { - if (context == null) { - return node; - } - let dependency = node.value; - - if (assetGraph.isDependencySkipped(dependency)) { - actions.skipChildren(); - return node; - } - - invariant(context?.type === 'asset'); - let parentAsset = context.value; - - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return node; - } - - for (let childAsset of assets) { - if ( - dependency.priority === 'lazy' || - childAsset.bundleBehavior === 'isolated' // An isolated Dependency, or Bundle must contain all assets it needs to load. - ) { - let bundleId = bundles.get(childAsset.id); - let bundle; - if (bundleId == null) { - let firstBundleGroup = nullthrows( - bundleGraph.getNode(stack[0][1]), - ); - invariant(firstBundleGroup !== 'root'); - bundle = createBundle({ - asset: childAsset, - target: firstBundleGroup.target, - needsStableName: - dependency.bundleBehavior === 'inline' || - childAsset.bundleBehavior === 'inline' - ? false - : dependency.isEntry || dependency.needsStableName, - bundleBehavior: - dependency.bundleBehavior ?? childAsset.bundleBehavior, - }); - bundleId = bundleGraph.addNode(bundle); - bundles.set(childAsset.id, bundleId); - bundleRoots.set(childAsset, [bundleId, bundleId]); - bundleGroupBundleIds.add(bundleId); - bundleGraph.addEdge(bundleGraphRootNodeId, bundleId); - } else { - bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); - - if ( - // If this dependency requests isolated, but the bundle is not, - // make the bundle isolated for all uses. - dependency.bundleBehavior === 'isolated' && - bundle.bundleBehavior == null - ) { - bundle.bundleBehavior = dependency.bundleBehavior; - } - } - - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded(dependency.id, { - value: dependency, - type: 'dependency', - }), - dependencyBundleGraph.addNodeByContentKeyIfNeeded( - String(bundleId), - { - value: bundle, - type: 'bundle', - }, - ), - dependencyPriorityEdges[dependency.priority], - ); - continue; - } - if ( - parentAsset.type !== childAsset.type || - dependency.priority === 'parallel' || - childAsset.bundleBehavior === 'inline' - ) { - // The referencing bundleRoot is the root of a Bundle that first brings in another bundle (essentially the FIRST parent of a bundle, this may or may not be a bundleGroup) - let [referencingBundleRoot, bundleGroupNodeId] = nullthrows( - stack[stack.length - 1], - ); - let bundleGroup = nullthrows( - bundleGraph.getNode(bundleGroupNodeId), - ); - invariant(bundleGroup !== 'root'); - - let bundleId; - let referencingBundleId = nullthrows( - bundleRoots.get(referencingBundleRoot), - )[0]; - let referencingBundle = nullthrows( - bundleGraph.getNode(referencingBundleId), - ); - invariant(referencingBundle !== 'root'); - let bundle; - bundleId = bundles.get(childAsset.id); - - /** - * If this is an entry bundlegroup, we only allow one bundle per type in those groups - * So attempt to add the asset to the entry bundle if it's of the same type. - * This asset will be created by other dependency if it's in another bundlegroup - * and bundles of other types should be merged in the next step - */ - let bundleGroupRootAsset = nullthrows(bundleGroup.mainEntryAsset); - if ( - entries.has(bundleGroupRootAsset) && - canMerge(bundleGroupRootAsset, childAsset) && - dependency.bundleBehavior == null - ) { - bundleId = bundleGroupNodeId; - } - if (bundleId == null) { - bundle = createBundle({ - // Bundles created from type changes shouldn't have an entry asset. - asset: childAsset, - type: childAsset.type, - env: childAsset.env, - bundleBehavior: - dependency.bundleBehavior ?? childAsset.bundleBehavior, - target: referencingBundle.target, - needsStableName: - childAsset.bundleBehavior === 'inline' || - dependency.bundleBehavior === 'inline' || - (dependency.priority === 'parallel' && - !dependency.needsStableName) - ? false - : referencingBundle.needsStableName, - }); - bundleId = bundleGraph.addNode(bundle); - - // Store Type-Change bundles for later since we need to know ALL bundlegroups they are part of to reduce/combine them - if (parentAsset.type !== childAsset.type) { - typeChangeIds.add(bundleId); - } - } else { - bundle = bundleGraph.getNode(bundleId); - invariant(bundle != null && bundle !== 'root'); - - if ( - // If this dependency requests isolated, but the bundle is not, - // make the bundle isolated for all uses. - dependency.bundleBehavior === 'isolated' && - bundle.bundleBehavior == null - ) { - bundle.bundleBehavior = dependency.bundleBehavior; - } - } - - bundles.set(childAsset.id, bundleId); - - // A bundle can belong to multiple bundlegroups, all the bundle groups of it's - // ancestors, and all async and entry bundles before it are "bundle groups" - // TODO: We may need to track bundles to all bundleGroups it belongs to in the future. - bundleRoots.set(childAsset, [bundleId, bundleGroupNodeId]); - bundleGraph.addEdge(referencingBundleId, bundleId); - - if (bundleId != bundleGroupNodeId) { - dependencyBundleGraph.addEdge( - dependencyBundleGraph.addNodeByContentKeyIfNeeded( - dependency.id, - { - value: dependency, - type: 'dependency', - }, - ), - dependencyBundleGraph.addNodeByContentKeyIfNeeded( - String(bundleId), - { - value: bundle, - type: 'bundle', - }, - ), - dependencyPriorityEdges.parallel, - ); - } - - assetReference.get(childAsset).push([dependency, bundle]); - continue; - } - } - } - return node; - }, - exit(node) { - if (stack[stack.length - 1]?.[0] === node.value) { - stack.pop(); - } - }, - }); - - // Step Merge Type Change Bundles: Clean up type change bundles within the exact same bundlegroups - for (let [nodeIdA, a] of bundleGraph.nodes) { - //if bundle b bundlegroups ==== bundle a bundlegroups then combine type changes - if (!typeChangeIds.has(nodeIdA) || a === 'root') continue; - let bundleABundleGroups = getBundleGroupsForBundle(nodeIdA); - for (let [nodeIdB, b] of bundleGraph.nodes) { - if ( - a !== 'root' && - b !== 'root' && - a !== b && - typeChangeIds.has(nodeIdB) && - canMerge(a, b) - ) { - let bundleBbundleGroups = getBundleGroupsForBundle(nodeIdB); - if (setEqual(bundleBbundleGroups, bundleABundleGroups)) { - let shouldMerge = true; - for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( - dependencyBundleGraph.getNodeIdByContentKey(String(nodeIdB)), - ALL_EDGE_TYPES, - )) { - let depNode = dependencyBundleGraph.getNode(depId); - // Cannot merge Dependency URL specifier type - if ( - depNode && - depNode.type === 'dependency' && - depNode.value.specifierType === 'url' - ) { - shouldMerge = false; - continue; - } - } - if (!shouldMerge) continue; - mergeBundle(nodeIdA, nodeIdB); - } - } - } - } - - /** - * Step Determine Reachability: Determine reachability for every asset from each bundleRoot. - * This is later used to determine which bundles to place each asset in. We build up two - * structures, one traversal each. ReachableRoots to store sync relationships, - * and bundleRootGraph to store the minimal availability through `parallel` and `async` relationships. - * The two graphs, are used to build up ancestorAssets, a structure which holds all availability by - * all means for each asset. - */ - for (let [root] of bundleRoots) { - if (!entries.has(root)) { - bundleRootGraph.addNodeByContentKey(root.id, root); // Add in all bundleRoots to BundleRootGraph - } - } - // ReachableRoots is a Graph of Asset Nodes which represents a BundleRoot, to all assets (non-bundleroot assets - // available to it synchronously (directly) built by traversing the assetgraph once. - for (let [root] of bundleRoots) { - // Add sync relationships to ReachableRoots - let rootNodeId = reachableRoots.addNodeByContentKeyIfNeeded(root.id, root); - assetGraph.traverse((node, _, actions) => { - if (node.value === root) { - return; - } - if (node.type === 'dependency') { - let dependency = node.value; - - if (dependencyBundleGraph.hasContentKey(dependency.id)) { - if (dependency.priority !== 'sync') { - let assets = assetGraph.getDependencyAssets(dependency); - if (assets.length === 0) { - return; - } - invariant(assets.length === 1); - let bundleRoot = assets[0]; - let bundle = nullthrows( - bundleGraph.getNode(nullthrows(bundles.get(bundleRoot.id))), - ); - if ( - bundle !== 'root' && - bundle.bundleBehavior == null && - !bundle.env.isIsolated() && - bundle.env.context === root.env.context - ) { - bundleRootGraph.addEdge( - bundleRootGraph.getNodeIdByContentKey(root.id), - bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), - dependency.priority === 'parallel' - ? bundleRootEdgeTypes.parallel - : bundleRootEdgeTypes.lazy, - ); - } - } - } - - if (dependency.priority !== 'sync') { - actions.skipChildren(); - } - return; - } - //asset node type - let asset = node.value; - if (asset.bundleBehavior != null || root.type !== asset.type) { - actions.skipChildren(); - return; - } - let nodeId = reachableRoots.addNodeByContentKeyIfNeeded( - node.value.id, - node.value, - ); - reachableRoots.addEdge(rootNodeId, nodeId); - }, root); - } - // Maps a given bundleRoot to the assets reachable from it, - // and the bundleRoots reachable from each of these assets - let ancestorAssets: Map> = new Map(); - - for (let entry of entries.keys()) { - // Initialize an empty set of ancestors available to entries - ancestorAssets.set(entry, new Set()); - } - - // Step Determine Availability - // Visit nodes in a topological order, visiting parent nodes before child nodes. - - // This allows us to construct an understanding of which assets will already be - // loaded and available when a bundle runs, by pushing available assets downwards and - // computing the intersection of assets available through all possible paths to a bundle. - // We call this structure ancestorAssets, a Map that tracks a bundleRoot, - // to all assets available to it (meaning they will exist guaranteed when the bundleRoot is loaded) - // The topological sort ensures all parents are visited before the node we want to process. - for (let nodeId of bundleRootGraph.topoSort(ALL_EDGE_TYPES)) { - const bundleRoot = bundleRootGraph.getNode(nodeId); - if (bundleRoot === 'root') continue; - invariant(bundleRoot != null); - let bundleGroupId = nullthrows(bundleRoots.get(bundleRoot))[1]; - - // At a BundleRoot, we access it's available assets (via ancestorAssets), - // and add to that all assets within the bundles in that BundleGroup. - - // This set is available to all bundles in a particular bundleGroup because - // bundleGroups are just bundles loaded at the same time. However it is - // not true that a bundle's available assets = all assets of all the bundleGroups - // it belongs to. It's the intersection of those sets. - let available; - if (bundleRoot.bundleBehavior === 'isolated') { - available = new Set(); - } else { - available = new Set(ancestorAssets.get(bundleRoot)); - for (let bundleIdInGroup of [ - bundleGroupId, - ...bundleGraph.getNodeIdsConnectedFrom(bundleGroupId), - ]) { - let bundleInGroup = nullthrows(bundleGraph.getNode(bundleIdInGroup)); - invariant(bundleInGroup !== 'root'); - if (bundleInGroup.bundleBehavior != null) { - continue; - } - - for (let bundleRoot of bundleInGroup.assets) { - // Assets directly connected to current bundleRoot - let assetsFromBundleRoot = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); - - for (let asset of [bundleRoot, ...assetsFromBundleRoot]) { - available.add(asset); - } - } - } - } - - // Now that we have bundleGroup availability, we will propagate that down to all the children - // of this bundleGroup. For a child, we also must maintain parallel availability. If it has - // parallel siblings that come before it, those, too, are available to it. Add those parallel - // available assets to the set of available assets for this child as well. - let children = bundleRootGraph.getNodeIdsConnectedFrom( - nodeId, - ALL_EDGE_TYPES, - ); - let parallelAvailability: Set = new Set(); - - for (let childId of children) { - let child = bundleRootGraph.getNode(childId); - invariant(child !== 'root' && child != null); - let bundleBehavior = getBundleFromBundleRoot(child).bundleBehavior; - if (bundleBehavior != null) { - continue; - } - let isParallel = bundleRootGraph.hasEdge( - nodeId, - childId, - bundleRootEdgeTypes.parallel, - ); - - // Most of the time, a child will have many parent bundleGroups, - // so the next time we peek at a child from another parent, we will - // intersect the availability built there with the previously computed - // availability. this ensures no matter which bundleGroup loads a particular bundle, - // it will only assume availability of assets it has under any circumstance - const childAvailableAssets = ancestorAssets.get(child); - let currentChildAvailable = isParallel - ? setUnion(parallelAvailability, available) - : available; - if (childAvailableAssets != null) { - setIntersect(childAvailableAssets, currentChildAvailable); - } else { - ancestorAssets.set(child, new Set(currentChildAvailable)); - } - if (isParallel) { - let assetsFromBundleRoot = reachableRoots - .getNodeIdsConnectedFrom( - reachableRoots.getNodeIdByContentKey(child.id), - ) - .map(id => nullthrows(reachableRoots.getNode(id))); - parallelAvailability = setUnion( - parallelAvailability, - assetsFromBundleRoot, - ); - parallelAvailability.add(child); //The next sibling should have older sibling available via parallel - } - } - } - // Step Internalize async bundles - internalize Async bundles if and only if, - // the bundle is synchronously available elsewhere. - // We can query sync assets available via reachableRoots. If the parent has - // the bundleRoot by reachableRoots AND ancestorAssets, internalize it. - for (let [id, bundleRoot] of bundleRootGraph.nodes) { - if (bundleRoot === 'root') continue; - let parentRoots = bundleRootGraph - .getNodeIdsConnectedTo(id, ALL_EDGE_TYPES) - .map(id => nullthrows(bundleRootGraph.getNode(id))); - let canDelete = - getBundleFromBundleRoot(bundleRoot).bundleBehavior !== 'isolated'; - if (parentRoots.length === 0) continue; - for (let parent of parentRoots) { - if (parent === 'root') { - canDelete = false; - continue; - } - if ( - reachableRoots.hasEdge( - reachableRoots.getNodeIdByContentKey(parent.id), - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - ) || - ancestorAssets.get(parent)?.has(bundleRoot) - ) { - let parentBundle = bundleGraph.getNode( - nullthrows(bundles.get(parent.id)), - ); - invariant(parentBundle != null && parentBundle !== 'root'); - parentBundle.internalizedAssetIds.push(bundleRoot.id); - } else { - canDelete = false; - } - } - if (canDelete) { - deleteBundle(bundleRoot); - } - } - // Step Insert Or Share: Place all assets into bundles or create shared bundles. Each asset - // is placed into a single bundle based on the bundle entries it is reachable from. - // This creates a maximally code split bundle graph with no duplication. - for (let asset of assets) { - // Unreliable bundleRoot assets which need to pulled in by shared bundles or other means - let reachable: Array = getReachableBundleRoots( - asset, - reachableRoots, - ).reverse(); - - let reachableEntries = []; - let reachableNonEntries = []; - - // Filter out entries, since they can't have shared bundles. - // Neither can non-splittable, isolated, or needing of stable name bundles. - // Reserve those filtered out bundles since we add the asset back into them. - for (let a of reachable) { - if ( - entries.has(a) || - !a.isBundleSplittable || - getBundleFromBundleRoot(a).needsStableName || - getBundleFromBundleRoot(a).bundleBehavior === 'isolated' - ) { - reachableEntries.push(a); - } else { - reachableNonEntries.push(a); - } - } - reachable = reachableNonEntries; - - // Filter out bundles from this asset's reachable array if - // bundle does not contain the asset in its ancestry - reachable = reachable.filter(b => !ancestorAssets.get(b)?.has(asset)); - - // Finally, filter out bundleRoots (bundles) from this assets - // reachable if they are subgraphs, and reuse that subgraph bundle - // by drawing an edge. Essentially, if two bundles within an asset's - // reachable array, have an ancestor-subgraph relationship, draw that edge. - // This allows for us to reuse a bundle instead of making a shared bundle if - // a bundle represents the exact set of assets a set of bundles would share - - // if a bundle b is a subgraph of another bundle f, reuse it, drawing an edge between the two - let canReuse: Set = new Set(); - for (let candidateSourceBundleRoot of reachable) { - let candidateSourceBundleId = nullthrows( - bundles.get(candidateSourceBundleRoot.id), - ); - if (candidateSourceBundleRoot.env.isIsolated()) { - continue; - } - let reuseableBundleId = bundles.get(asset.id); - if (reuseableBundleId != null) { - canReuse.add(candidateSourceBundleRoot); - bundleGraph.addEdge(candidateSourceBundleId, reuseableBundleId); - - let reusableBundle = bundleGraph.getNode(reuseableBundleId); - invariant(reusableBundle !== 'root' && reusableBundle != null); - reusableBundle.sourceBundles.add(candidateSourceBundleId); - } else { - // Asset is not a bundleRoot, but if its ancestor bundle (in the asset's reachable) can be - // reused as a subgraph of another bundleRoot in its reachable, reuse it - for (let otherReuseCandidate of reachable) { - if (candidateSourceBundleRoot === otherReuseCandidate) continue; - let reusableCandidateReachable = getReachableBundleRoots( - otherReuseCandidate, - reachableRoots, - ).filter(b => !ancestorAssets.get(b)?.has(otherReuseCandidate)); - if (reusableCandidateReachable.includes(candidateSourceBundleRoot)) { - let reusableBundleId = nullthrows( - bundles.get(otherReuseCandidate.id), - ); - canReuse.add(candidateSourceBundleRoot); - bundleGraph.addEdge( - nullthrows(bundles.get(candidateSourceBundleRoot.id)), - reusableBundleId, - ); - let reusableBundle = bundleGraph.getNode(reusableBundleId); - invariant(reusableBundle !== 'root' && reusableBundle != null); - reusableBundle.sourceBundles.add(candidateSourceBundleId); - } - } - } - } - //Bundles that are reused should not be considered for shared bundles, so filter them out - reachable = reachable.filter(b => !canReuse.has(b)); - - // Add assets to non-splittable bundles. - for (let entry of reachableEntries) { - let entryBundleId = nullthrows(bundles.get(entry.id)); - let entryBundle = nullthrows(bundleGraph.getNode(entryBundleId)); - invariant(entryBundle !== 'root'); - entryBundle.assets.add(asset); - entryBundle.size += asset.stats.size; - } - - // Create shared bundles for splittable bundles. - if (reachable.length > config.minBundles) { - let sourceBundles = reachable.map(a => nullthrows(bundles.get(a.id))); - let key = reachable.map(a => a.id).join(','); - let bundleId = bundles.get(key); - let bundle; - if (bundleId == null) { - let firstSourceBundle = nullthrows( - bundleGraph.getNode(sourceBundles[0]), - ); - invariant(firstSourceBundle !== 'root'); - bundle = createBundle({ - target: firstSourceBundle.target, - type: firstSourceBundle.type, - env: firstSourceBundle.env, - }); - bundle.sourceBundles = new Set(sourceBundles); - let sharedInternalizedAssets = new Set( - firstSourceBundle.internalizedAssetIds, - ); - - for (let p of sourceBundles) { - let parentBundle = nullthrows(bundleGraph.getNode(p)); - invariant(parentBundle !== 'root'); - if (parentBundle === firstSourceBundle) continue; - setIntersect( - sharedInternalizedAssets, - new Set(parentBundle.internalizedAssetIds), - ); - } - bundle.internalizedAssetIds = [...sharedInternalizedAssets]; - bundleId = bundleGraph.addNode(bundle); - bundles.set(key, bundleId); - } else { - bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); - } - bundle.assets.add(asset); - bundle.size += asset.stats.size; - - for (let sourceBundleId of sourceBundles) { - if (bundleId !== sourceBundleId) { - bundleGraph.addEdge(sourceBundleId, bundleId); - } - } - - dependencyBundleGraph.addNodeByContentKeyIfNeeded(String(bundleId), { - value: bundle, - type: 'bundle', - }); - } else if (reachable.length <= config.minBundles) { - for (let root of reachable) { - let bundle = nullthrows( - bundleGraph.getNode(nullthrows(bundles.get(root.id))), - ); - invariant(bundle !== 'root'); - bundle.assets.add(asset); - bundle.size += asset.stats.size; - } - } - } - // Step Merge Share Bundles: Merge any shared bundles under the minimum bundle size back into - // their source bundles, and remove the bundle. - // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained - for (let [bundleNodeId, bundle] of bundleGraph.nodes) { - if (bundle === 'root') continue; - if ( - bundle.sourceBundles.size > 0 && - bundle.mainEntryAsset == null && - bundle.size < config.minBundleSize - ) { - removeBundle(bundleGraph, bundleNodeId, assetReference); - } - } - - // Step Remove Shared Bundles: Remove shared bundles from bundle groups that hit the parallel request limit. - for (let bundleGroupId of bundleGraph.getNodeIdsConnectedFrom(rootNodeId)) { - // Find shared bundles in this bundle group. - let bundleId = bundleGroupId; - - // We should include "bundle reuse" as shared bundles that may be removed but the bundle itself would have to be retained - let bundleIdsInGroup = getBundlesForBundleGroup(bundleId); //get all bundlegrups this bundle is an ancestor of - if (bundleIdsInGroup.length > config.maxParallelRequests) { - let sharedBundleIdsInBundleGroup = bundleIdsInGroup.filter(b => { - let bundle = nullthrows(bundleGraph.getNode(b)); - // shared bundles must have source bundles, we could have a bundle - // connected to another bundle that isnt a shared bundle, so check - return ( - bundle !== 'root' && bundle.sourceBundles.size > 0 && bundleId != b - ); - }); - - let numBundlesInGroup = bundleIdsInGroup.length; - // Sort the bundles so the smallest ones are removed first. - let sharedBundlesInGroup = sharedBundleIdsInBundleGroup - .map(id => ({ - id, - bundle: nullthrows(bundleGraph.getNode(id)), - })) - .map(({id, bundle}) => { - // For Flow - invariant(bundle !== 'root'); - return {id, bundle}; - }) - .sort((a, b) => b.bundle.size - a.bundle.size); - - // Remove bundles until the bundle group is within the parallel request limit. - while ( - sharedBundlesInGroup.length > 0 && - numBundlesInGroup > config.maxParallelRequests - ) { - let bundleTuple = sharedBundlesInGroup.pop(); - let bundleToRemove = bundleTuple.bundle; - let bundleIdToRemove = bundleTuple.id; - //TODO add integration test where bundles in bunlde group > max parallel request limit & only remove a couple shared bundles - // but total # bundles still exceeds limit due to non shared bundles - - // Add all assets in the shared bundle into the source bundles that are within this bundle group. - let sourceBundles = [...bundleToRemove.sourceBundles].filter(b => - bundleIdsInGroup.includes(b), - ); - - for (let sourceBundleId of sourceBundles) { - let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); - invariant(sourceBundle !== 'root'); - bundleToRemove.sourceBundles.delete(sourceBundleId); - for (let asset of bundleToRemove.assets) { - sourceBundle.assets.add(asset); - sourceBundle.size += asset.stats.size; - } - //This case is specific to reused bundles, which can have shared bundles attached to it - for (let childId of bundleGraph.getNodeIdsConnectedFrom( - bundleIdToRemove, - )) { - let child = bundleGraph.getNode(childId); - invariant(child !== 'root' && child != null); - child.sourceBundles.add(sourceBundleId); - bundleGraph.addEdge(sourceBundleId, childId); - } - // needs to add test case where shared bundle is removed from ONE bundlegroup but not from the whole graph! - // Remove the edge from this bundle group to the shared bundle. - // If there is now only a single bundle group that contains this bundle, - // merge it into the remaining source bundles. If it is orphaned entirely, remove it. - let incomingNodeCount = - bundleGraph.getNodeIdsConnectedTo(bundleIdToRemove).length; - - if ( - incomingNodeCount <= 2 && - //Never fully remove reused bundles - bundleToRemove.mainEntryAsset == null - ) { - // If one bundle group removes a shared bundle, but the other *can* keep it, still remove because that shared bundle is pointless (only one source bundle) - removeBundle(bundleGraph, bundleIdToRemove, assetReference); - // Stop iterating through bundleToRemove's sourceBundles as the bundle has been removed. - break; - } else { - bundleGraph.removeEdge(sourceBundleId, bundleIdToRemove); - } - } - numBundlesInGroup--; - } - } - } - function deleteBundle(bundleRoot: BundleRoot) { - bundleGraph.removeNode(nullthrows(bundles.get(bundleRoot.id))); - bundleRoots.delete(bundleRoot); - bundles.delete(bundleRoot.id); - if (reachableRoots.hasContentKey(bundleRoot.id)) { - reachableRoots.replaceNodeIdsConnectedTo( - reachableRoots.getNodeIdByContentKey(bundleRoot.id), - [], - ); - } - if (bundleRootGraph.hasContentKey(bundleRoot.id)) { - bundleRootGraph.removeNode( - bundleRootGraph.getNodeIdByContentKey(bundleRoot.id), - ); - } - } - function getBundleGroupsForBundle(nodeId: NodeId) { - let bundleGroupBundleIds = new Set(); - bundleGraph.traverseAncestors(nodeId, ancestorId => { - if ( - bundleGraph - .getNodeIdsConnectedTo(ancestorId) //if node is root, then dont add, otherwise do add. - .includes(bundleGraph.rootNodeId) - ) { - bundleGroupBundleIds.add(ancestorId); - } - }); - return bundleGroupBundleIds; - } - function getBundlesForBundleGroup(bundleGroupId) { - let bundlesInABundleGroup = []; - bundleGraph.traverse(nodeId => { - bundlesInABundleGroup.push(nodeId); - }, bundleGroupId); - return bundlesInABundleGroup; - } - - function mergeBundle(mainNodeId: NodeId, otherNodeId: NodeId) { - //merges assets of "otherRoot" into "mainBundleRoot" - let a = nullthrows(bundleGraph.getNode(mainNodeId)); - let b = nullthrows(bundleGraph.getNode(otherNodeId)); - invariant(a !== 'root' && b !== 'root'); - let bundleRootB = nullthrows(b.mainEntryAsset); - let mainBundleRoot = nullthrows(a.mainEntryAsset); - for (let asset of a.assets) { - b.assets.add(asset); - } - a.assets = b.assets; - for (let depId of dependencyBundleGraph.getNodeIdsConnectedTo( - dependencyBundleGraph.getNodeIdByContentKey(String(otherNodeId)), - ALL_EDGE_TYPES, - )) { - dependencyBundleGraph.replaceNodeIdsConnectedTo(depId, [ - dependencyBundleGraph.getNodeIdByContentKey(String(mainNodeId)), - ]); - } - - //clean up asset reference - for (let dependencyTuple of assetReference.get(bundleRootB)) { - dependencyTuple[1] = a; - } - //add in any lost edges - for (let nodeId of bundleGraph.getNodeIdsConnectedTo(otherNodeId)) { - bundleGraph.addEdge(nodeId, mainNodeId); - } - deleteBundle(bundleRootB); - let bundleGroupOfMain = nullthrows(bundleRoots.get(mainBundleRoot))[1]; - bundleRoots.set(bundleRootB, [mainNodeId, bundleGroupOfMain]); - bundles.set(bundleRootB.id, mainNodeId); - } - function getBundleFromBundleRoot(bundleRoot: BundleRoot): Bundle { - let bundle = bundleGraph.getNode( - nullthrows(bundleRoots.get(bundleRoot))[0], - ); - invariant(bundle !== 'root' && bundle != null); - return bundle; - } - - return { - bundleGraph, - dependencyBundleGraph, - bundleGroupBundleIds, - assetReference, - }; -} - -const CONFIG_SCHEMA: SchemaEntity = { - type: 'object', - properties: { - http: { - type: 'number', - enum: Object.keys(HTTP_OPTIONS).map(k => Number(k)), - }, - minBundles: { - type: 'number', - }, - minBundleSize: { - type: 'number', - }, - maxParallelRequests: { - type: 'number', - }, - }, - additionalProperties: false, -}; - -function createBundle(opts: {| - uniqueKey?: string, - target: Target, - asset?: Asset, - env?: Environment, - type?: string, - needsStableName?: boolean, - bundleBehavior?: ?BundleBehavior, -|}): Bundle { - if (opts.asset == null) { - return { - uniqueKey: opts.uniqueKey, - assets: new Set(), - internalizedAssetIds: [], - mainEntryAsset: null, - size: 0, - sourceBundles: new Set(), - target: opts.target, - type: nullthrows(opts.type), - env: nullthrows(opts.env), - needsStableName: Boolean(opts.needsStableName), - bundleBehavior: opts.bundleBehavior, - }; - } - - let asset = nullthrows(opts.asset); - return { - uniqueKey: opts.uniqueKey, - assets: new Set([asset]), - internalizedAssetIds: [], - mainEntryAsset: asset, - size: asset.stats.size, - sourceBundles: new Set(), - target: opts.target, - type: opts.type ?? asset.type, - env: opts.env ?? asset.env, - needsStableName: Boolean(opts.needsStableName), - bundleBehavior: opts.bundleBehavior ?? asset.bundleBehavior, - }; -} - -function removeBundle( - bundleGraph: Graph, - bundleId: NodeId, - assetReference: DefaultMap>, -) { - let bundle = nullthrows(bundleGraph.getNode(bundleId)); - invariant(bundle !== 'root'); - for (let asset of bundle.assets) { - assetReference.set( - asset, - assetReference.get(asset).filter(t => !t.includes(bundle)), - ); - for (let sourceBundleId of bundle.sourceBundles) { - let sourceBundle = nullthrows(bundleGraph.getNode(sourceBundleId)); - invariant(sourceBundle !== 'root'); - sourceBundle.assets.add(asset); - sourceBundle.size += asset.stats.size; - } - } - - bundleGraph.removeNode(bundleId); -} - -async function loadBundlerConfig( - config: Config, - options: PluginOptions, -): Promise { - let conf = await config.getConfig([], { - packageKey: '@parcel/bundler-default', - }); - - if (!conf) { - return HTTP_OPTIONS['2']; - } - - invariant(conf?.contents != null); - - validateSchema.diagnostic( - CONFIG_SCHEMA, - { - data: conf?.contents, - source: await options.inputFS.readFile(conf.filePath, 'utf8'), - filePath: conf.filePath, - prependKey: `/${encodeJSONKeyComponent('@parcel/bundler-default')}`, - }, - '@parcel/bundler-experimental', - 'Invalid config for @parcel/bundler-experimental', - ); - - let http = conf.contents.http ?? 2; - let defaults = HTTP_OPTIONS[http]; - - return { - minBundles: conf.contents.minBundles ?? defaults.minBundles, - minBundleSize: conf.contents.minBundleSize ?? defaults.minBundleSize, - maxParallelRequests: - conf.contents.maxParallelRequests ?? defaults.maxParallelRequests, - }; -} - -function getReachableBundleRoots(asset, graph): Array { - return graph - .getNodeIdsConnectedTo(graph.getNodeIdByContentKey(asset.id)) - .map(nodeId => nullthrows(graph.getNode(nodeId))); -} - -function getEntryByTarget( - bundleGraph: MutableBundleGraph, -): DefaultMap> { - // Find entries from assetGraph per target - let targets: DefaultMap> = new DefaultMap( - () => new Map(), - ); - bundleGraph.traverse({ - enter(node, context, actions) { - if (node.type !== 'asset') { - return node; - } - invariant( - context != null && - context.type === 'dependency' && - context.value.isEntry && - context.value.target != null, - ); - targets.get(context.value.target.distDir).set(node.value, context.value); - actions.skipChildren(); - return node; - }, - }); - return targets; -} - -function canMerge(a, b) { - // Bundles can be merged if they have the same type and environment, - // unless they are explicitly marked as isolated or inline. - return ( - a.type === b.type && - a.env.context === b.env.context && - a.bundleBehavior == null && - b.bundleBehavior == null - ); -} diff --git a/packages/core/integration-tests/test/bundler.js b/packages/core/integration-tests/test/bundler.js index 498ec7a80fe..c750e67f843 100644 --- a/packages/core/integration-tests/test/bundler.js +++ b/packages/core/integration-tests/test/bundler.js @@ -51,53 +51,51 @@ describe('bundler', function () { }); it('should remove reused bundle (over shared bundles based on size) if the bundlegroup hit the parallel request limit', async function () { - if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { - let b = await bundle( - path.join( - __dirname, - 'integration/shared-bundle-reused-bundle-remove-reuse/index.js', - ), - { - mode: 'production', - defaultTargetOptions: { - shouldScopeHoist: false, - }, + let b = await bundle( + path.join( + __dirname, + 'integration/shared-bundle-reused-bundle-remove-reuse/index.js', + ), + { + mode: 'production', + defaultTargetOptions: { + shouldScopeHoist: false, }, - ); + }, + ); - assertBundles(b, [ - { - name: 'index.js', - assets: [ - 'index.js', - 'bundle-url.js', - 'cacheLoader.js', - 'css-loader.js', - 'esmodule-helpers.js', - 'js-loader.js', - 'bundle-manifest.js', - ], - }, - { - assets: ['bar.js', 'foo.js', 'a.js', 'b.js'], - }, - { - assets: ['buzz.js'], - }, - { - assets: ['c.js'], - }, - { - assets: ['a.js', 'b.js', 'foo.js'], - }, - { - assets: ['styles.css'], - }, - { - assets: ['local.html'], - }, - ]); - } + assertBundles(b, [ + { + name: 'index.js', + assets: [ + 'index.js', + 'bundle-url.js', + 'cacheLoader.js', + 'css-loader.js', + 'esmodule-helpers.js', + 'js-loader.js', + 'bundle-manifest.js', + ], + }, + { + assets: ['bar.js', 'foo.js', 'a.js', 'b.js'], + }, + { + assets: ['buzz.js'], + }, + { + assets: ['c.js'], + }, + { + assets: ['a.js', 'b.js', 'foo.js'], + }, + { + assets: ['styles.css'], + }, + { + assets: ['local.html'], + }, + ]); }); //This test case is the same as previous except we remove the shared bundle since it is smaller @@ -205,60 +203,56 @@ describe('bundler', function () { //The shared bundle should only be 'put back' for the bundlegroups which hit the parallel request limit // But if there are at least two other bundlegroups using this shared bundle that do not hit the max limit // the shared bundle should not be removed from the graph - if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { - let b = await bundle( - path.join( - __dirname, - 'integration/shared-bundle-between-reused-bundle-removal/index.js', - ), - { - mode: 'production', - defaultTargetOptions: { - shouldScopeHoist: false, - }, + let b = await bundle( + path.join( + __dirname, + 'integration/shared-bundle-between-reused-bundle-removal/index.js', + ), + { + mode: 'production', + defaultTargetOptions: { + shouldScopeHoist: false, }, - ); + }, + ); - assertBundles(b, [ - { - name: 'index.js', - assets: [ - 'index.js', - 'bundle-url.js', - 'cacheLoader.js', - 'css-loader.js', - 'esmodule-helpers.js', - 'js-loader.js', - 'bundle-manifest.js', - ], - }, - { - assets: ['bar.js', 'foo.js', 'a.js', 'b.js'], // shared bundle merged back - }, - { - assets: ['buzz.js'], - }, - { - assets: ['c.js'], // shared bundle - }, - { - assets: ['foo.js', 'a.js', 'b.js'], - }, - { - assets: ['styles.css'], - }, - { - assets: ['local.html'], - }, - ]); + assertBundles(b, [ + { + name: 'index.js', + assets: [ + 'index.js', + 'bundle-url.js', + 'cacheLoader.js', + 'css-loader.js', + 'esmodule-helpers.js', + 'js-loader.js', + 'bundle-manifest.js', + ], + }, + { + assets: ['bar.js', 'foo.js', 'a.js', 'b.js'], // shared bundle merged back + }, + { + assets: ['buzz.js'], + }, + { + assets: ['c.js'], // shared bundle + }, + { + assets: ['foo.js', 'a.js', 'b.js'], + }, + { + assets: ['styles.css'], + }, + { + assets: ['local.html'], + }, + ]); - assert( - b - .getReferencedBundles( - b.getBundlesWithAsset(findAsset(b, 'bar.js'))[0], - ) - .includes(b.getBundlesWithAsset(findAsset(b, 'c.js'))[0]), - ); - } + assert( + b + .getReferencedBundles(b.getBundlesWithAsset(findAsset(b, 'bar.js'))[0]) + .includes(b.getBundlesWithAsset(findAsset(b, 'c.js'))[0]), + ); }); }); diff --git a/packages/core/integration-tests/test/css-modules.js b/packages/core/integration-tests/test/css-modules.js index b3c96bc6793..9d2b1dc84c5 100644 --- a/packages/core/integration-tests/test/css-modules.js +++ b/packages/core/integration-tests/test/css-modules.js @@ -577,77 +577,42 @@ describe('css modules', () => { assert.deepEqual(res, [['page2', '_4fY2uG_foo _1ZEqVW_foo j1UkRG_foo']]); - if (process.env.PARCEL_TEST_EXPERIMENTAL_BUNDLER) { - assertBundles(b, [ - { - name: 'page1.html', - assets: ['page1.html'], - }, - { - name: 'page2.html', - assets: ['page2.html'], - }, - { - type: 'js', - assets: [ - 'page1.js', - 'index.module.css', - 'a.module.css', - 'b.module.css', - ], - }, - { - type: 'js', - assets: [ - 'page2.js', - 'index.module.css', - 'a.module.css', - 'b.module.css', - ], - }, - { - type: 'css', - assets: ['a.module.css', 'b.module.css'], - }, - { - type: 'css', - assets: ['index.module.css'], - }, - ]); - } else { - assertBundles(b, [ - { - name: 'page1.html', - assets: ['page1.html'], - }, - { - name: 'page2.html', - assets: ['page2.html'], - }, - { - type: 'js', - assets: [ - 'page1.js', - 'index.module.css', - 'a.module.css', - 'b.module.css', - ], - }, - { - type: 'js', - assets: [ - 'page2.js', - 'index.module.css', - 'a.module.css', - 'b.module.css', - ], - }, - { - type: 'css', - assets: ['index.module.css', 'a.module.css', 'b.module.css'], - }, - ]); - } + assertBundles(b, [ + { + name: 'page1.html', + assets: ['page1.html'], + }, + { + name: 'page2.html', + assets: ['page2.html'], + }, + { + type: 'js', + assets: [ + 'page1.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'js', + assets: [ + 'page2.js', + 'index.module.css', + 'a.module.css', + 'b.module.css', + ], + }, + { + type: 'css', + assets: ['a.module.css', 'b.module.css'], + }, + { + type: 'css', + assets: ['index.module.css'], + }, + ]); }); it('should not process inline