Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Cache invalidations for packagers when reading other bundles #8370

Merged
merged 10 commits into from Sep 7, 2022
Merged

Cache invalidations for packagers when reading other bundles #8370

Show file tree
Hide file tree
Changes from 8 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
abc984a
Different API for config loading
mischnic Aug 5, 2022
792963d
Rename loadBundleConfig
mischnic Aug 24, 2022
1971271
Pass in public config
mischnic Aug 24, 2022
52e1d32
invalidateOnBuild
mischnic Aug 24, 2022
c465483
Typed runPackage invocation
mischnic Aug 24, 2022
84385ae
Add test
mischnic Aug 31, 2022
0f80a57
Fixup invalidations
mischnic Aug 31, 2022
a301c37
Cleanup
mischnic Aug 31, 2022
ad2e110
Fix variable name
mischnic Sep 6, 2022
c7624a9
Merge branch 'v2' into loadGlobalInfo
mischnic Sep 6, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions packages/core/core/src/InternalConfig.js
View file
Open in desktop
Expand Up @@ -25,6 +25,7 @@ type ConfigOpts = {|
invalidateOnOptionChange?: Set<string>,
devDeps?: Array<InternalDevDepOptions>,
invalidateOnStartup?: boolean,
invalidateOnBuild?: boolean,
|};

export function createConfig({
Expand All @@ -39,6 +40,7 @@ export function createConfig({
invalidateOnOptionChange,
devDeps,
invalidateOnStartup,
invalidateOnBuild,
}: ConfigOpts): Config {
let environment = env ?? createEnvironment();
return {
Expand All @@ -59,5 +61,6 @@ export function createConfig({
invalidateOnOptionChange: invalidateOnOptionChange ?? new Set(),
devDeps: devDeps ?? [],
invalidateOnStartup: invalidateOnStartup ?? false,
invalidateOnBuild: invalidateOnBuild ?? false,
};
}
171 changes: 126 additions & 45 deletions packages/core/core/src/PackagerRunner.js
View file
Open in desktop
Expand Up @@ -38,6 +38,7 @@ import BundleGraph, {
bundleGraphToInternalBundleGraph,
} from './public/BundleGraph';
import PluginOptions from './public/PluginOptions';
import PublicConfig from './public/Config';
import {PARCEL_VERSION, HASH_REF_PREFIX, HASH_REF_REGEX} from './constants';
import {
fromProjectPath,
Expand All @@ -50,7 +51,7 @@ import {
loadPluginConfig,
getConfigHash,
getConfigRequests,
type PluginWithLoadConfig,
type PluginWithLoadConfigGlobalInfo,
} from './requests/ConfigRequest';
import {
createDevDependency,
Expand Down Expand Up @@ -143,12 +144,20 @@ export default class PackagerRunner {
): Promise<PackageRequestResult> {
invalidateDevDeps(invalidDevDeps, this.options, this.config);

let configs = await this.loadConfigs(bundleGraph, bundle);
let {configs, bundleConfigs} = await this.loadConfigs(bundleGraph, bundle);
let bundleInfo =
(await this.getBundleInfoFromCache(bundleGraph, bundle, configs)) ??
(await this.getBundleInfo(bundle, bundleGraph, configs));

let configRequests = getConfigRequests([...configs.values()]);
(await this.getBundleInfoFromCache(
bundleGraph,
bundle,
configs,
bundleConfigs,
)) ??
(await this.getBundleInfo(bundle, bundleGraph, configs, bundleConfigs));

let configRequests = getConfigRequests([
...configs.values(),
...bundleConfigs.values(),
]);
let devDepRequests = getWorkerDevDepRequests([
...this.devDepRequests.values(),
]);
Expand All @@ -164,77 +173,118 @@ export default class PackagerRunner {
async loadConfigs(
bundleGraph: InternalBundleGraph,
bundle: InternalBundle,
): Promise<Map<string, Config>> {
): Promise<{|
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
|}> {
let configs = new Map();
let bundleConfigs = new Map();

await this.loadConfig(bundle, configs);
await this.loadConfig(bundleGraph, bundle, configs, bundleConfigs);
for (let inlineBundle of bundleGraph.getInlineBundles(bundle)) {
await this.loadConfig(inlineBundle, configs);
await this.loadConfig(bundleGraph, inlineBundle, configs, bundleConfigs);
}

return configs;
return {configs, bundleConfigs};
}

async loadConfig(
bundleGraph: InternalBundleGraph,
bundle: InternalBundle,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
): Promise<void> {
let name = nullthrows(bundle.name);
let plugin = await this.config.getPackager(name);
await this.loadPluginConfig(plugin, configs);
await this.loadPluginConfig(
bundleGraph,
bundle,
plugin,
configs,
bundleConfigs,
);

let optimizers = await this.config.getOptimizers(name, bundle.pipeline);

for (let optimizer of optimizers) {
await this.loadPluginConfig(optimizer, configs);
await this.loadPluginConfig(
bundleGraph,
bundle,
optimizer,
configs,
bundleConfigs,
);
}
}

async loadPluginConfig<T: PluginWithLoadConfig>(
async loadPluginConfig<T: PluginWithLoadConfigGlobalInfo>(
bundleGraph: InternalBundleGraph,
bundle: InternalBundle,
plugin: LoadedPlugin<T>,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
): Promise<void> {
if (configs.has(plugin.name)) {
return;
}
if (!configs.has(plugin.name)) {
// Only load config for a plugin once per build.
let existing = pluginConfigs.get(plugin.name);
if (existing != null) {
configs.set(plugin.name, existing);
} else {
if (plugin.plugin.loadConfig != null) {
let config = createConfig({
plugin: plugin.name,
searchPath: toProjectPathUnsafe('index'),
});

await loadPluginConfig(plugin, config, this.options);

for (let devDep of config.devDeps) {
let devDepRequest = await createDevDependency(
devDep,
this.previousDevDeps,
this.options,
);
let key = `${devDep.specifier}:${fromProjectPath(
this.options.projectRoot,
devDep.resolveFrom,
)}`;
this.devDepRequests.set(key, devDepRequest);
}

// Only load config for a plugin once per build.
let existing = pluginConfigs.get(plugin.name);
if (existing != null) {
configs.set(plugin.name, existing);
return;
pluginConfigs.set(plugin.name, config);
configs.set(plugin.name, config);
}
}
}

if (plugin.plugin.loadConfig != null) {
let loadBundleConfig = plugin.plugin.loadBundleConfig;
if (!bundleConfigs.has(plugin.name) && loadBundleConfig != null) {
let config = createConfig({
plugin: plugin.name,
searchPath: toProjectPathUnsafe('index'),
searchPath: joinProjectPath(
bundle.target.distDir,
bundle.name ?? bundle.id,
),
Comment on lines +263 to +266
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think if name is null, then it's an inline bundle? Not sure what the searchPath should be in that case

});

await loadPluginConfig(plugin, config, this.options);

for (let devDep of config.devDeps) {
let devDepRequest = await createDevDependency(
devDep,
this.previousDevDeps,
config.result = await loadBundleConfig({
bundle: NamedBundle.get(bundle, bundleGraph, this.options),
bundleGraph: new BundleGraph<NamedBundleType>(
bundleGraph,
NamedBundle.get.bind(NamedBundle),
this.options,
);
let key = `${devDep.specifier}:${fromProjectPath(
this.options.projectRoot,
devDep.resolveFrom,
)}`;
this.devDepRequests.set(key, devDepRequest);
}

pluginConfigs.set(plugin.name, config);
configs.set(plugin.name, config);
),
config: new PublicConfig(config, this.options),
options: new PluginOptions(this.options),
logger: new PluginLogger({origin: plugin.name}),
});
bundleConfigs.set(plugin.name, config);
}
}

async getBundleInfoFromCache(
bundleGraph: InternalBundleGraph,
bundle: InternalBundle,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
): Async<?BundleInfo> {
if (this.options.shouldDisableCache) {
return;
Expand All @@ -244,6 +294,7 @@ export default class PackagerRunner {
bundle,
bundleGraph,
configs,
bundleConfigs,
this.previousInvalidations,
);
let infoKey = PackagerRunner.getInfoKey(cacheKey);
Expand All @@ -254,17 +305,23 @@ export default class PackagerRunner {
bundle: InternalBundle,
bundleGraph: InternalBundleGraph,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
): Promise<BundleInfo> {
let {type, contents, map} = await this.getBundleResult(
bundle,
bundleGraph,
configs,
bundleConfigs,
);

// Recompute cache keys as they may have changed due to dev dependencies.
let cacheKey = await this.getCacheKey(bundle, bundleGraph, configs, [
...this.invalidations.values(),
]);
let cacheKey = await this.getCacheKey(
bundle,
bundleGraph,
configs,
bundleConfigs,
[...this.invalidations.values()],
);
let cacheKeys = {
content: PackagerRunner.getContentKey(cacheKey),
map: PackagerRunner.getMapKey(cacheKey),
Expand All @@ -278,12 +335,18 @@ export default class PackagerRunner {
bundle: InternalBundle,
bundleGraph: InternalBundleGraph,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
): Promise<{|
type: string,
contents: Blob,
map: ?string,
|}> {
let packaged = await this.package(bundle, bundleGraph, configs);
let packaged = await this.package(
bundle,
bundleGraph,
configs,
bundleConfigs,
);
let type = packaged.type ?? bundle.type;
let res = await this.optimize(
bundle,
Expand All @@ -292,6 +355,7 @@ export default class PackagerRunner {
packaged.contents,
packaged.map,
configs,
bundleConfigs,
);

let map =
Expand Down Expand Up @@ -319,6 +383,7 @@ export default class PackagerRunner {
internalBundle: InternalBundle,
bundleGraph: InternalBundleGraph,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
): Promise<BundleResult> {
let bundle = NamedBundle.get(internalBundle, bundleGraph, this.options);
this.report({
Expand All @@ -332,6 +397,7 @@ export default class PackagerRunner {
try {
return await plugin.package({
config: configs.get(name)?.result,
bundleConfig: bundleConfigs.get(name)?.result,
bundle,
bundleGraph: new BundleGraph<NamedBundleType>(
bundleGraph,
Expand All @@ -358,6 +424,7 @@ export default class PackagerRunner {
// $FlowFixMe
bundleGraphToInternalBundleGraph(bundleGraph),
configs,
bundleConfigs,
);

return {contents: res.contents};
Expand Down Expand Up @@ -395,6 +462,7 @@ export default class PackagerRunner {
contents: Blob,
map?: ?SourceMap,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
): Promise<BundleResult> {
let bundle = NamedBundle.get(
internalBundle,
Expand Down Expand Up @@ -430,6 +498,7 @@ export default class PackagerRunner {
try {
let next = await optimizer.plugin.optimize({
config: configs.get(optimizer.name)?.result,
bundleConfig: bundleConfigs.get(optimizer.name)?.result,
bundle,
bundleGraph,
contents: optimized.contents,
Expand Down Expand Up @@ -535,6 +604,7 @@ export default class PackagerRunner {
bundle: InternalBundle,
bundleGraph: InternalBundleGraph,
configs: Map<string, Config>,
bundleConfigs: Map<string, Config>,
invalidations: Array<RequestInvalidation>,
): Promise<string> {
let configResults = {};
Expand All @@ -547,6 +617,16 @@ export default class PackagerRunner {
);
}
}
let globalInfoResults = {};
for (let [pluginName, config] of bundleConfigs) {
if (config) {
globalInfoResults[pluginName] = await getConfigHash(
config,
pluginName,
this.options,
);
}
}

let devDepHashes = await this.getDevDepHashes(bundle);
for (let inlineBundle of bundleGraph.getInlineBundles(bundle)) {
Expand All @@ -565,6 +645,7 @@ export default class PackagerRunner {
bundle.target.publicUrl +
bundleGraph.getHash(bundle) +
JSON.stringify(configResults) +
JSON.stringify(globalInfoResults) +
this.options.mode,
);
}
Expand Down
2 changes: 2 additions & 0 deletions packages/core/core/src/Parcel.js
View file
Open in desktop
Expand Up @@ -261,6 +261,8 @@ export default class Parcel {
type: 'buildStart',
});

this.#requestTracker.graph.invalidateOnBuildNodes();

let request = createParcelBuildRequest({
optionsRef: this.#optionsRef,
requestedAssetIds: this.#requestedAssetIds,
Expand Down
8 changes: 4 additions & 4 deletions packages/core/core/src/ParcelConfig.js
View file
Open in desktop
Expand Up @@ -253,7 +253,7 @@ export default class ParcelConfig {

async getPackager(
filePath: FilePath,
): Promise<LoadedPlugin<Packager<mixed>>> {
): Promise<LoadedPlugin<Packager<mixed, mixed>>> {
let packager = this.matchGlobMap(
toProjectPathUnsafe(filePath),
this.packagers,
Expand All @@ -265,7 +265,7 @@ export default class ParcelConfig {
'/packagers',
);
}
return this.loadPlugin<Packager<mixed>>(packager);
return this.loadPlugin<Packager<mixed, mixed>>(packager);
}

_getOptimizerNodes(
Expand Down Expand Up @@ -298,13 +298,13 @@ export default class ParcelConfig {
getOptimizers(
filePath: FilePath,
pipeline: ?string,
): Promise<Array<LoadedPlugin<Optimizer<mixed>>>> {
): Promise<Array<LoadedPlugin<Optimizer<mixed, mixed>>>> {
let optimizers = this._getOptimizerNodes(filePath, pipeline);
if (optimizers.length === 0) {
return Promise.resolve([]);
}

return this.loadPlugins<Optimizer<mixed>>(optimizers);
return this.loadPlugins<Optimizer<mixed, mixed>>(optimizers);
}

async getCompressors(
Expand Down