Add moduleParsed plugin hook (#3813)

* Attach ESTree AST to all Nodes and do not expose internal AST

* Include code and AST in module info

* Add moduleParsed hook

* Add documentation

cli/help.md

@@ -32,6 +32,7 @@ Basic options:
 --exports <mode>            Specify export mode (auto, default, named, none)
 --extend                    Extend global variable defined by --name
 --no-externalLiveBindings   Do not generate code to support live bindings
+--failAfterWarnings         Exit with an error if the build produced warnings
 --footer <text>             Code to insert at end of bundle (outside wrapper)
 --no-freeze                 Do not freeze namespace objects
 --no-hoistTransitiveImports Do not hoist transitive imports into entry chunks
@@ -46,14 +47,13 @@ Basic options:
 --preferConst               Use `const` instead of `var` for exports
 --no-preserveEntrySignatures Avoid facade chunks for entry points
 --preserveModules           Preserve module structure
---preserveModulesRoot       Preserved modules under this path are rooted in output `dir`
+--preserveModulesRoot       Put preserved modules under this path at root level
 --preserveSymlinks          Do not follow symlinks when resolving files
 --shimMissingExports        Create shim variables for missing exports
 --silent                    Don't print warnings
---failAfterWarnings         Exit with an error code if there was a warning during the build
 --sourcemapExcludeSources   Do not include source code in source maps
 --sourcemapFile <file>      Specify bundle position for source maps
---stdin=ext                 Specify file extension used for stdin input - default is none
+--stdin=ext                 Specify file extension used for stdin input
 --no-stdin                  Do not read "-" from stdin
 --no-strict                 Don't emit `"use strict";` in the generated modules
 --strictDeprecations        Throw errors for deprecated features

docs/01-command-line-reference.md

@@ -290,6 +290,7 @@ Many options have command line equivalents. In those cases, any arguments passed
 --exports <mode>            Specify export mode (auto, default, named, none)
 --extend                    Extend global variable defined by --name
 --no-externalLiveBindings   Do not generate code to support live bindings
+--failAfterWarnings         Exit with an error if the build produced warnings
 --footer <text>             Code to insert at end of bundle (outside wrapper)
 --no-freeze                 Do not freeze namespace objects
 --no-hoistTransitiveImports Do not hoist transitive imports into entry chunks
@@ -304,14 +305,13 @@ Many options have command line equivalents. In those cases, any arguments passed
 --preferConst               Use `const` instead of `var` for exports
 --no-preserveEntrySignatures Avoid facade chunks for entry points
 --preserveModules           Preserve module structure
---preserveModulesRoot       Preserved modules under this path are rooted in output `dir`
+--preserveModulesRoot       Put preserved modules under this path at root level
 --preserveSymlinks          Do not follow symlinks when resolving files
 --shimMissingExports        Create shim variables for missing exports
 --silent                    Don't print warnings
---failAfterWarnings         Exit with an error code if there was a warning during the build
 --sourcemapExcludeSources   Do not include source code in source maps
 --sourcemapFile <file>      Specify bundle position for source maps
---stdin=ext                 Specify file extension used for stdin input - default is none
+--stdin=ext                 Specify file extension used for stdin input
 --no-stdin                  Do not read "-" from stdin
 --no-strict                 Don't emit `"use strict";` in the generated modules
 --strictDeprecations        Throw errors for deprecated features

docs/05-plugin-development.md

@@ -79,7 +79,7 @@ See [Output Generation Hooks](guide/en/#output-generation-hooks) for hooks that
 #### `buildEnd`
 Type: `(error?: Error) => void`<br>
 Kind: `async, parallel`<br>
-Previous Hook: [`transform`](guide/en/#transform), [`resolveId`](guide/en/#resolveid) or [`resolveDynamicImport`](guide/en/#resolvedynamicimport).<br>
+Previous Hook: [`moduleParsed`](guide/en/#moduleparsed), [`resolveId`](guide/en/#resolveid) or [`resolveDynamicImport`](guide/en/#resolvedynamicimport).<br>
 Next Hook: [`outputOptions`](guide/en/#outputoptions) in the output generation phase as this is the last hook of the build phase.
 
 Called when rollup has finished bundling, but before `generate` or `write` is called; you can also return a Promise. If an error occurred during the build, it is passed on to this hook.
@@ -108,6 +108,18 @@ See [custom module meta-data](guide/en/#custom-module-meta-data) for how to use
 
 You can use [`this.getModuleInfo`](guide/en/#thisgetmoduleinfomoduleid-string--moduleinfo--null) to find out the previous values of `moduleSideEffects`, `syntheticNamedExports` and `meta` inside this hook.
 
+#### `moduleParsed`
+Type: `(moduleInfo: ModuleInfo) => void`<br>
+Kind: `async, parallel`<br>
+Previous Hook: [`transform`](guide/en/#transform) where the currently handled file was transformed.<br>
+NextHook: [`resolveId`](guide/en/#resolveid) and [`resolveDynamicImport`](guide/en/#resolvedynamicimport) to resolve all discovered static and dynamic imports in parallel if present, otherwise [`buildEnd`](guide/en/#buildend).
+
+This hook is called each time a module has been fully parsed by Rollup. See [`this.getModuleInfo`](guide/en/#thisgetmoduleinfomoduleid-string--moduleinfo--null) for what information is passed to this hook.
+
+In contrast to the [`transform`](guide/en/#transform) hook, this hook is never cached and can be used to get information about both cached and other modules, including the final shape of the `meta` property, the `code` and the `ast`.
+
+Note that information about imported modules is not yet available in this hook, and information about importing modules may be incomplete as additional importers could be discovered later. If you need this information, use the [`buildEnd`](guide/en/#buildend) hook.
+
 #### `options`
 Type: `(options: InputOptions) => InputOptions | null`<br>
 Kind: `async, sequential`<br>
@@ -121,7 +133,7 @@ This is the only hook that does not have access to most [plugin context](guide/e
 #### `resolveDynamicImport`
 Type: `(specifier: string | ESTree.Node, importer: string) => string | false | null | {id: string, external?: boolean}`<br>
 Kind: `async, first`<br>
-Previous Hook: [`transform`](guide/en/#transform) where the importing file was transformed.<br>
+Previous Hook: [`moduleParsed`](guide/en/#moduleparsed) for the importing file.<br>
 Next Hook: [`load`](guide/en/#load) if the hook resolved with an id that has not yet been loaded, [`resolveId`](guide/en/#resolveid) if the dynamic import contains a string and was not resolved by the hook, otherwise [`buildEnd`](guide/en/#buildend).
 
 Defines a custom resolver for dynamic imports. Returning `false` signals that the import should be kept as it is and not be passed to other resolvers thus making it external. Similar to the [`resolveId`](guide/en/#resolveid) hook, you can also return an object to resolve the import to a different id while marking it as external at the same time.
@@ -138,7 +150,7 @@ Note that the return value of this hook will not be passed to `resolveId` afterw
 #### `resolveId`
 Type: `(source: string, importer: string | undefined, options: {custom?: {[plugin: string]: any}) => string | false | null | {id: string, external?: boolean, moduleSideEffects?: boolean | "no-treeshake" | null, syntheticNamedExports?: boolean | string | null, meta?: {[plugin: string]: any} | null}`<br>
 Kind: `async, first`<br>
-Previous Hook: [`buildStart`](guide/en/#buildstart) if we are resolving an entry point, [`transform`](guide/en/#transform) if we are resolving an import, or as fallback for [`resolveDynamicImport`](guide/en/#resolvedynamicimport). Additionally this hook can be triggered during the build phase from plugin hooks by calling [`this.emitFile`](guide/en/#thisemitfileemittedfile-emittedchunk--emittedasset--string) to emit an entry point or at any time by calling [`this.resolve`](guide/en/#thisresolvesource-string-importer-string-options-skipself-boolean-custom-plugin-string-any--promiseid-string-external-boolean-modulesideeffects-boolean--no-treeshake-syntheticnamedexports-boolean--string-custom-plugin-string-any--null) to manually resolve an id.<br>
+Previous Hook: [`buildStart`](guide/en/#buildstart) if we are resolving an entry point, [`moduleParsed`](guide/en/#moduleparsed) if we are resolving an import, or as fallback for [`resolveDynamicImport`](guide/en/#resolvedynamicimport). Additionally this hook can be triggered during the build phase from plugin hooks by calling [`this.emitFile`](guide/en/#thisemitfileemittedfile-emittedchunk--emittedasset--string) to emit an entry point or at any time by calling [`this.resolve`](guide/en/#thisresolvesource-string-importer-string-options-skipself-boolean-custom-plugin-string-any--promiseid-string-external-boolean-modulesideeffects-boolean--no-treeshake-syntheticnamedexports-boolean--string-custom-plugin-string-any--null) to manually resolve an id.<br>
 Next Hook: [`load`](guide/en/#load) if the resolved id that has not yet been loaded, otherwise [`buildEnd`](guide/en/#buildend).
 
 Defines a custom resolver. A resolver can be useful for e.g. locating third-party dependencies. Here `source` is the importee exactly as it is written in the import statement, i.e. for
@@ -201,7 +213,7 @@ When triggering this hook from a plugin via [`this.resolve(source, importer, opt
 Type: `(code: string, id: string) => string | null | {code?: string, map?: string | SourceMap, ast? : ESTree.Program, moduleSideEffects?: boolean | "no-treeshake" | null, syntheticNamedExports?: boolean | string | null, meta?: {[plugin: string]: any} | null}`<br>
 Kind: `async, sequential`<br>
 Previous Hook: [`load`](guide/en/#load) where the currently handled file was loaded.<br>
-NextHook: [`resolveId`](guide/en/#resolveid) and [`resolveDynamicImport`](guide/en/#resolvedynamicimport) to resolve all discovered static and dynamic imports in parallel if present, otherwise [`buildEnd`](guide/en/#buildend).
+NextHook: [`moduleParsed`](guide/en/#moduleparsed) once the file has been processed and parsed.
 
 Can be used to transform individual modules. To prevent additional parsing overhead in case e.g. this hook already used `this.parse` to generate an AST for some reason, this hook can optionally return a `{ code, ast, map }` object. The `ast` must be a standard ESTree AST with `start` and `end` properties for each node. If the transformation does not move code, you can preserve existing sourcemaps by setting `map` to `null`. Otherwise you might need to generate the source map. See [the section on source code transformations](#source-code-transformations).
 
@@ -624,6 +636,8 @@ Returns additional information about the module in question in the form
 ```
 {
   id: string, // the id of the module, for convenience
+  code: string | null, // the source code of the module, `null` if external or not yet available
+  ast: ESTree.Program, // the parsed abstract syntax tree if available
   isEntry: boolean, // is this a user- or plugin-defined entry point
   isExternal: boolean, // for external modules that are referenced but not included in the graph
   importedIds: string[], // the module ids statically imported by this module
@@ -636,7 +650,10 @@ Returns additional information about the module in question in the form
 }
 ```
 
-This utility function returns `null` if the module id cannot be found.
+During the build, this object represents currently available information about the module. Before the [`buildEnd`](guide/en/#buildend) hook, this information may be incomplete as e.g.
+ the `importedIds` are not yet resolved or additional `importers` are discovered.
+ 
+Returns `null` if the module id cannot be found.
 
 #### `this.meta: {rollupVersion: string, watchMode: boolean}`
 

src/Bundle.ts

@@ -267,7 +267,7 @@ function getIncludedModules(modulesById: Map<string, Module | ExternalModule>):
 	return [...modulesById.values()].filter(
 		module =>
 			module instanceof Module &&
-			(module.isIncluded() || module.isEntryPoint || module.includedDynamicImporters.length > 0)
+			(module.isIncluded() || module.info.isEntry || module.includedDynamicImporters.length > 0)
 	) as Module[];
 }
 

src/Chunk.ts

@@ -162,7 +162,7 @@ export default class Chunk {
 		}
 		if (
 			!chunk.dependencies.has(chunkByModule.get(facadedModule)!) &&
-			facadedModule.moduleSideEffects &&
+			facadedModule.info.hasModuleSideEffects &&
 			facadedModule.hasEffects()
 		) {
 			chunk.dependencies.add(chunkByModule.get(facadedModule)!);
@@ -234,7 +234,7 @@ export default class Chunk {
 			if (this.isEmpty && module.isIncluded()) {
 				this.isEmpty = false;
 			}
-			if (module.isEntryPoint || outputOptions.preserveModules) {
+			if (module.info.isEntry || outputOptions.preserveModules) {
 				this.entryModules.push(module);
 			}
 			for (const importer of module.includedDynamicImporters) {
@@ -307,7 +307,7 @@ export default class Chunk {
 		} else {
 			assignExportsToNames(remainingExports, this.exportsByName, this.exportNamesByVariable);
 		}
-		if (this.outputOptions.preserveModules || (this.facadeModule && this.facadeModule.isEntryPoint))
+		if (this.outputOptions.preserveModules || (this.facadeModule && this.facadeModule.info.isEntry))
 			this.exportMode = getExportMode(
 				this,
 				this.outputOptions,
@@ -473,7 +473,7 @@ export default class Chunk {
 			exports: this.getExportNames(),
 			facadeModuleId: facadeModule && facadeModule.id,
 			isDynamicEntry: this.dynamicEntryModules.length > 0,
-			isEntry: facadeModule !== null && facadeModule.isEntryPoint,
+			isEntry: facadeModule !== null && facadeModule.info.isEntry,
 			isImplicitEntry: this.implicitEntryModules.length > 0,
 			modules: this.renderedModules,
 			get name() {
@@ -718,7 +718,7 @@ export default class Chunk {
 				intro: addons.intro!,
 				isEntryModuleFacade:
 					this.outputOptions.preserveModules ||
-					(this.facadeModule !== null && this.facadeModule.isEntryPoint),
+					(this.facadeModule !== null && this.facadeModule.info.isEntry),
 				namedExportsMode: this.exportMode !== 'default',
 				outro: addons.outro!,
 				usesTopLevelAwait,
@@ -1307,7 +1307,7 @@ export default class Chunk {
 		}
 		if (
 			this.includedNamespaces.has(module) ||
-			(module.isEntryPoint && module.preserveSignature !== false) ||
+			(module.info.isEntry && module.preserveSignature !== false) ||
 			module.includedDynamicImporters.some(importer => this.chunkByModule.get(importer) !== this)
 		) {
 			this.ensureReexportsAreAvailableForModule(module);

src/ExternalModule.ts

@@ -1,9 +1,11 @@
 import ExternalVariable from './ast/variables/ExternalVariable';
 import {
 	CustomPluginOptions,
+	ModuleInfo,
 	NormalizedInputOptions,
 	NormalizedOutputOptions
 } from './rollup/types';
+import { EMPTY_ARRAY } from './utils/blank';
 import { makeLegal } from './utils/identifierHelpers';
 import { isAbsolute, normalize, relative } from './utils/path';
 
@@ -15,6 +17,7 @@ export default class ExternalModule {
 	execIndex: number;
 	exportedVariables: Map<ExternalVariable, string>;
 	importers: string[] = [];
+	info: ModuleInfo;
 	mostCommonSuggestion = 0;
 	namespaceVariableName = '';
 	nameSuggestions: { [name: string]: number };
@@ -28,19 +31,35 @@ export default class ExternalModule {
 	constructor(
 		private readonly options: NormalizedInputOptions,
 		public readonly id: string,
-		public moduleSideEffects: boolean | 'no-treeshake',
-		public meta: CustomPluginOptions
+		hasModuleSideEffects: boolean | 'no-treeshake',
+		meta: CustomPluginOptions
 	) {
-		this.id = id;
 		this.execIndex = Infinity;
-		this.moduleSideEffects = moduleSideEffects;
-
-		const parts = id.split(/[\\/]/);
-		this.suggestedVariableName = makeLegal(parts.pop()!);
-
+		this.suggestedVariableName = makeLegal(id.split(/[\\/]/).pop()!);
 		this.nameSuggestions = Object.create(null);
 		this.declarations = Object.create(null);
 		this.exportedVariables = new Map();
+
+		const module = this;
+		this.info = {
+			ast: null,
+			code: null,
+			dynamicallyImportedIds: EMPTY_ARRAY,
+			get dynamicImporters() {
+				return module.dynamicImporters.sort();
+			},
+			hasModuleSideEffects,
+			id,
+			implicitlyLoadedAfterOneOf: EMPTY_ARRAY,
+			implicitlyLoadedBefore: EMPTY_ARRAY,
+			importedIds: EMPTY_ARRAY,
+			get importers() {
+				return module.importers.sort();
+			},
+			isEntry: false,
+			isExternal: true,
+			meta
+		};
 	}
 
 	getVariableForExportName(name: string): ExternalVariable {

src/Graph.ts

@@ -12,11 +12,9 @@ import {
 	RollupWatcher,
 	SerializablePluginCache
 } from './rollup/types';
-import { EMPTY_ARRAY } from './utils/blank';
 import { BuildPhase } from './utils/buildPhase';
 import { errImplicitDependantIsNotIncluded, error } from './utils/error';
 import { analyseModuleExecution } from './utils/executionOrder';
-import { getId } from './utils/getId';
 import { PluginDriver } from './utils/PluginDriver';
 import relativeId from './utils/relativeId';
 import { timeEnd, timeStart } from './utils/timers';
@@ -136,47 +134,7 @@ export default class Graph {
 	getModuleInfo = (moduleId: string): ModuleInfo | null => {
 		const foundModule = this.modulesById.get(moduleId);
 		if (!foundModule) return null;
-		return {
-			get dynamicallyImportedIds() {
-				if (foundModule instanceof Module) {
-					const dynamicallyImportedIds: string[] = [];
-					for (const { resolution } of foundModule.dynamicImports) {
-						if (resolution instanceof Module || resolution instanceof ExternalModule) {
-							dynamicallyImportedIds.push(resolution.id);
-						}
-					}
-					return dynamicallyImportedIds;
-				}
-				return EMPTY_ARRAY;
-			},
-			get dynamicImporters() {
-				return foundModule!.dynamicImporters.sort();
-			},
-			hasModuleSideEffects: foundModule.moduleSideEffects,
-			id: foundModule.id,
-			get implicitlyLoadedAfterOneOf() {
-				return foundModule instanceof Module
-					? Array.from(foundModule.implicitlyLoadedAfter, getId)
-					: EMPTY_ARRAY;
-			},
-			get implicitlyLoadedBefore() {
-				return foundModule instanceof Module
-					? Array.from(foundModule.implicitlyLoadedBefore, getId)
-					: [];
-			},
-			get importedIds() {
-				if (foundModule instanceof Module) {
-					return Array.from(foundModule.sources, source => foundModule.resolvedIds[source].id);
-				}
-				return EMPTY_ARRAY;
-			},
-			get importers() {
-				return foundModule!.importers.sort();
-			},
-			isEntry: foundModule instanceof Module && foundModule.isEntryPoint,
-			isExternal: foundModule instanceof ExternalModule,
-			meta: foundModule.meta
-		};
+		return foundModule.info;
 	};
 
 	private async generateModuleGraph(): Promise<void> {
@@ -211,7 +169,7 @@ export default class Graph {
 				this.needsTreeshakingPass = false;
 				for (const module of this.modules) {
 					if (module.isExecuted) {
-						if (module.moduleSideEffects === 'no-treeshake') {
+						if (module.info.hasModuleSideEffects === 'no-treeshake') {
 							module.includeAllInBundle();
 						} else {
 							module.include();
@@ -226,7 +184,7 @@ export default class Graph {
 		for (const externalModule of this.externalModules) externalModule.warnUnusedImports();
 		for (const module of this.implicitEntryModules) {
 			for (const dependant of module.implicitlyLoadedAfter) {
-				if (!(dependant.isEntryPoint || dependant.isIncluded())) {
+				if (!(dependant.info.isEntry || dependant.isIncluded())) {
 					error(errImplicitDependantIsNotIncluded(dependant));
 				}
 			}