[v3.0] Improve asset emission performance

docs/05-plugin-development.md

@@ -722,7 +722,7 @@ If there are no dynamic imports, this will create exactly three chunks where the
 
 Note that even though any module id can be used in `implicitlyLoadedAfterOneOf`, Rollup will throw an error if such an id cannot be uniquely associated with a chunk, e.g. because the `id` cannot be reached implicitly or explicitly from the existing static entry points, or because the file is completely tree-shaken. Using only entry points, either defined by the user or of previously emitted chunks, will always work, though.
 
-If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same source even if the `name` does not match. If such an asset is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used.
+If the `type` is _`asset`_, then this emits an arbitrary new file with the given `source` as content. It is possible to defer setting the `source` via [`this.setAssetSource(referenceId, source)`](guide/en/#thissetassetsource) to a later time to be able to reference a file during the build phase while setting the source separately for each output during the generate phase. Assets with a specified `fileName` will always generate separate files while other emitted assets may be deduplicated with existing assets if they have the same `string` source even if the `name` does not match. If the source is not a string but a typed array or `Buffer`, no deduplication will occur for performance reasons. If an asset without a `fileName` is not deduplicated, the [`output.assetFileNames`](guide/en/#outputassetfilenames) name pattern will be used.
 
 #### `this.error`
 

src/utils/FileEmitter.ts

@@ -61,7 +61,7 @@ function generateAssetFileName(
 
 function reserveFileNameInBundle(
 	fileName: string,
-	bundle: OutputBundleWithPlaceholders,
+	{ bundle }: FileEmitterOutput,
 	warn: WarningHandler
 ) {
 	if (bundle[lowercaseBundleKeys].has(fileName.toLowerCase())) {
@@ -151,11 +151,17 @@ function getChunkFileName(
 	return error(errChunkNotGeneratedForFileName(file.fileName || file.name));
 }
 
+interface FileEmitterOutput {
+	bundle: OutputBundleWithPlaceholders;
+	fileNamesBySource: Map<string, string>;
+	outputOptions: NormalizedOutputOptions;
+}
+
 export class FileEmitter {
-	private bundle: OutputBundleWithPlaceholders | null = null;
 	private facadeChunkByModule: ReadonlyMap<Module, Chunk> | null = null;
 	private readonly filesByReferenceId: Map<string, ConsumedFile>;
-	private outputOptions: NormalizedOutputOptions | null = null;
+	private nextIdBase = 1;
+	private output: FileEmitterOutput | null = null;
 
 	constructor(
 		private readonly graph: Graph,
@@ -222,8 +228,8 @@ export class FileEmitter {
 			return error(errAssetSourceAlreadySet(consumedFile.name || referenceId));
 		}
 		const source = getValidSource(requestedSource, consumedFile, referenceId);
-		if (this.bundle) {
-			this.finalizeAsset(consumedFile, source, referenceId, this.bundle);
+		if (this.output) {
+			this.finalizeAsset(consumedFile, source, referenceId, this.output);
 		} else {
 			consumedFile.source = source;
 		}
@@ -237,16 +243,19 @@ export class FileEmitter {
 		bundle: OutputBundleWithPlaceholders,
 		outputOptions: NormalizedOutputOptions
 	): void => {
-		this.outputOptions = outputOptions;
-		this.bundle = bundle;
+		const fileNamesBySource = new Map();
+		const output = (this.output = { bundle, fileNamesBySource, outputOptions });
 		for (const emittedFile of this.filesByReferenceId.values()) {
 			if (emittedFile.fileName) {
-				reserveFileNameInBundle(emittedFile.fileName, bundle, this.options.onwarn);
+				reserveFileNameInBundle(emittedFile.fileName, output, this.options.onwarn);
+				if (emittedFile.type === 'asset' && typeof emittedFile.source === 'string') {
+					fileNamesBySource.set(emittedFile.source, emittedFile.fileName);
+				}
 			}
 		}
 		for (const [referenceId, consumedFile] of this.filesByReferenceId) {
 			if (consumedFile.type === 'asset' && consumedFile.source !== undefined) {
-				this.finalizeAsset(consumedFile, consumedFile.source, referenceId, bundle);
+				this.finalizeAsset(consumedFile, consumedFile.source, referenceId, output);
 			}
 		}
 	};
@@ -278,14 +287,14 @@ export class FileEmitter {
 		};
 		const referenceId = this.assignReferenceId(
 			consumedAsset,
-			emittedAsset.fileName || emittedAsset.name || emittedAsset.type
+			emittedAsset.fileName || emittedAsset.name || String(this.nextIdBase++)
 		);
-		if (this.bundle) {
+		if (this.output) {
 			if (emittedAsset.fileName) {
-				reserveFileNameInBundle(emittedAsset.fileName, this.bundle, this.options.onwarn);
+				reserveFileNameInBundle(emittedAsset.fileName, this.output, this.options.onwarn);
 			}
 			if (source !== undefined) {
-				this.finalizeAsset(consumedAsset, source, referenceId, this.bundle);
+				this.finalizeAsset(consumedAsset, source, referenceId, this.output);
 			}
 		}
 		return referenceId;
@@ -323,16 +332,19 @@ export class FileEmitter {
 		consumedFile: ConsumedFile,
 		source: string | Uint8Array,
 		referenceId: string,
-		bundle: OutputBundleWithPlaceholders
+		{ bundle, fileNamesBySource, outputOptions }: FileEmitterOutput
 	): void {
 		const fileName =
 			consumedFile.fileName ||
-			findExistingAssetFileNameWithSource(bundle, source) ||
-			generateAssetFileName(consumedFile.name, source, this.outputOptions!, bundle);
+			(typeof source === 'string' && fileNamesBySource.get(source)) ||
+			generateAssetFileName(consumedFile.name, source, outputOptions, bundle);
 
 		// We must not modify the original assets to avoid interaction between outputs
 		const assetWithFileName = { ...consumedFile, fileName, source };
 		this.filesByReferenceId.set(referenceId, assetWithFileName);
+		if (typeof source === 'string') {
+			fileNamesBySource.set(source, fileName);
+		}
 		bundle[fileName] = {
 			fileName,
 			name: consumedFile.name,
@@ -341,40 +353,3 @@ export class FileEmitter {
 		};
 	}
 }
-
-// TODO This can lead to a performance problem when many assets are emitted.
-//  Instead, we should only deduplicate string assets and use their sources as
-//  object keys for better performance.
-function findExistingAssetFileNameWithSource(
-	bundle: OutputBundleWithPlaceholders,
-	source: string | Uint8Array
-): string | null {
-	for (const [fileName, outputFile] of Object.entries(bundle)) {
-		if (outputFile.type === 'asset' && areSourcesEqual(source, outputFile.source)) return fileName;
-	}
-	return null;
-}
-
-function areSourcesEqual(
-	sourceA: string | Uint8Array | Buffer,
-	sourceB: string | Uint8Array | Buffer
-): boolean {
-	if (typeof sourceA === 'string') {
-		return sourceA === sourceB;
-	}
-	if (typeof sourceB === 'string') {
-		return false;
-	}
-	if ('equals' in sourceA) {
-		return sourceA.equals(sourceB);
-	}
-	if (sourceA.length !== sourceB.length) {
-		return false;
-	}
-	for (let index = 0; index < sourceA.length; index++) {
-		if (sourceA[index] !== sourceB[index]) {
-			return false;
-		}
-	}
-	return true;
-}

test/chunking-form/samples/emit-file/deduplicate-assets/_config.js

@@ -4,26 +4,11 @@ module.exports = {
 		input: ['main.js'],
 		plugins: {
 			buildStart() {
-				this.emitFile({ type: 'asset', name: 'string.txt', source: 'hello world' });
-				this.emitFile({ type: 'asset', name: 'otherString.txt', source: 'hello world' });
-				this.emitFile({
-					type: 'asset',
-					name: 'buffer.txt',
-					source: Buffer.from([0x62, 0x75, 0x66, 0x66, 0x65, 0x72])
-				});
-				this.emitFile({
-					type: 'asset',
-					name: 'otherBuffer.txt',
-					source: Buffer.from([0x62, 0x75, 0x66, 0x66, 0x65, 0x72])
-				});
+				this.emitFile({ type: 'asset', name: 'string.txt', source: 'string' });
+				this.emitFile({ type: 'asset', name: 'otherString.txt', source: 'otherString' });
 
 				// specific file names will not be deduplicated
-				this.emitFile({ type: 'asset', fileName: 'named/string.txt', source: 'hello world' });
-				this.emitFile({
-					type: 'asset',
-					fileName: 'named/buffer.txt',
-					source: Buffer.from([0x62, 0x75, 0x66, 0x66, 0x65, 0x72])
-				});
+				this.emitFile({ type: 'asset', fileName: 'named/string.txt', source: 'named' });
 				return null;
 			}
 		}

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/otherString-e296c1ca.txt

@@ -0,0 +1 @@
+otherString

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/assets/string-473287f8.txt

@@ -0,0 +1 @@
+string

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/amd/named/string.txt

@@ -1 +1 @@
-hello world
+named

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/otherString-e296c1ca.txt

@@ -0,0 +1 @@
+otherString

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/assets/string-473287f8.txt

@@ -0,0 +1 @@
+string

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/cjs/named/string.txt

@@ -1 +1 @@
-hello world
+named

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/otherString-e296c1ca.txt

@@ -0,0 +1 @@
+otherString

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/assets/string-473287f8.txt

@@ -0,0 +1 @@
+string

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/es/named/string.txt

@@ -1 +1 @@
-hello world
+named

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/otherString-e296c1ca.txt

@@ -0,0 +1 @@
+otherString

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/assets/string-473287f8.txt

@@ -0,0 +1 @@
+string

test/chunking-form/samples/emit-file/deduplicate-assets/_expected/system/named/string.txt

@@ -1 +1 @@
-hello world
+named

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_config.js

@@ -5,54 +5,78 @@ module.exports = {
 		plugins: {
 			buildStart() {
 				this.emitFile({ type: 'asset', name: 'buildStart.txt', source: 'buildStart' });
-				this.emitFile({ type: 'asset', fileName: 'custom/buildStart.txt', source: 'buildStart' });
+				this.emitFile({
+					type: 'asset',
+					fileName: 'custom/buildStart.txt',
+					source: 'buildStart-custom'
+				});
 			},
 			resolveId() {
 				this.emitFile({ type: 'asset', name: 'resolveId.txt', source: 'resolveId' });
-				this.emitFile({ type: 'asset', fileName: 'custom/resolveId.txt', source: 'resolveId' });
+				this.emitFile({
+					type: 'asset',
+					fileName: 'custom/resolveId.txt',
+					source: 'resolveId-custom'
+				});
 			},
 			load() {
 				this.emitFile({ type: 'asset', name: 'load.txt', source: 'load' });
-				this.emitFile({ type: 'asset', fileName: 'custom/load.txt', source: 'load' });
+				this.emitFile({ type: 'asset', fileName: 'custom/load.txt', source: 'load-custom' });
 			},
 			transform() {
 				this.emitFile({ type: 'asset', name: 'transform.txt', source: 'transform' });
-				this.emitFile({ type: 'asset', fileName: 'custom/transform.txt', source: 'transform' });
+				this.emitFile({
+					type: 'asset',
+					fileName: 'custom/transform.txt',
+					source: 'transform-custom'
+				});
 			},
 			buildEnd() {
 				this.emitFile({ type: 'asset', name: 'buildEnd.txt', source: 'buildEnd' });
-				this.emitFile({ type: 'asset', fileName: 'custom/buildEnd.txt', source: 'buildEnd' });
+				this.emitFile({
+					type: 'asset',
+					fileName: 'custom/buildEnd.txt',
+					source: 'buildEnd-custom'
+				});
 			},
 			renderStart() {
 				this.emitFile({ type: 'asset', name: 'renderStart.txt', source: 'renderStart' });
-				this.emitFile({ type: 'asset', fileName: 'custom/renderStart.txt', source: 'renderStart' });
+				this.emitFile({
+					type: 'asset',
+					fileName: 'custom/renderStart.txt',
+					source: 'renderStart-custom'
+				});
 			},
 			banner() {
 				this.emitFile({ type: 'asset', name: 'banner.txt', source: 'banner' });
-				this.emitFile({ type: 'asset', fileName: 'custom/banner.txt', source: 'banner' });
+				this.emitFile({ type: 'asset', fileName: 'custom/banner.txt', source: 'banner-custom' });
 			},
 			footer() {
 				this.emitFile({ type: 'asset', name: 'footer.txt', source: 'footer' });
-				this.emitFile({ type: 'asset', fileName: 'custom/footer.txt', source: 'footer' });
+				this.emitFile({ type: 'asset', fileName: 'custom/footer.txt', source: 'footer-custom' });
 			},
 			intro() {
 				this.emitFile({ type: 'asset', name: 'intro.txt', source: 'intro' });
-				this.emitFile({ type: 'asset', fileName: 'custom/intro.txt', source: 'intro' });
+				this.emitFile({ type: 'asset', fileName: 'custom/intro.txt', source: 'intro-custom' });
 			},
 			outro() {
 				this.emitFile({ type: 'asset', name: 'outro.txt', source: 'outro' });
-				this.emitFile({ type: 'asset', fileName: 'custom/outro.txt', source: 'outro' });
+				this.emitFile({ type: 'asset', fileName: 'custom/outro.txt', source: 'outro-custom' });
 			},
 			renderChunk() {
 				this.emitFile({ type: 'asset', name: 'renderChunk.txt', source: 'renderChunk' });
-				this.emitFile({ type: 'asset', fileName: 'custom/renderChunk.txt', source: 'renderChunk' });
+				this.emitFile({
+					type: 'asset',
+					fileName: 'custom/renderChunk.txt',
+					source: 'renderChunk-custom'
+				});
 			},
 			generateBundle() {
 				this.emitFile({ type: 'asset', name: 'generateBundle.txt', source: 'generateBundle' });
 				this.emitFile({
 					type: 'asset',
 					fileName: 'custom/generateBundle.txt',
-					source: 'generateBundle'
+					source: 'generateBundle-custom'
 				});
 			}
 		}

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/banner.txt

@@ -1 +1 @@
-banner
+banner-custom

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/buildEnd.txt

@@ -1 +1 @@
-buildEnd
+buildEnd-custom

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/buildStart.txt

@@ -1 +1 @@
-buildStart
+buildStart-custom

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/footer.txt

@@ -1 +1 @@
-footer
+footer-custom

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/generateBundle.txt

@@ -1 +1 @@
-generateBundle
+generateBundle-custom

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/intro.txt

@@ -1 +1 @@
-intro
+intro-custom

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/load.txt

@@ -1 +1 @@
-load
+load-custom

test/chunking-form/samples/emit-file/emits-files-from-various-hooks/_expected/amd/custom/outro.txt

@@ -1 +1 @@
-outro
+outro-custom