Skip to content

Commit

Permalink
feat: persistent cache between compilations (webpack@5 only) (#541)
Browse files Browse the repository at this point in the history
  • Loading branch information
@evilebottnawi
evilebottnawi committed Oct 22, 2020
1 parent 93936a0 commit c892451
Show file tree
Hide file tree
Showing 4 changed files with 279 additions and 84 deletions.
1 change: 1 addition & 0 deletions .gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ npm-debug.log*
/reports
/node_modules
/test/fixtures/\[special\$directory\]
/test/outputs

.DS_Store
Thumbs.db
Expand Down
208 changes: 155 additions & 53 deletions src/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,58 @@ class CopyPlugin {
this.options = options.options || {};
}

static async createSnapshot(compilation, startTime, dependency) {
if (!compilation.fileSystemInfo) {
return;
}

// eslint-disable-next-line consistent-return
return new Promise((resolve, reject) => {
compilation.fileSystemInfo.createSnapshot(
startTime,
[dependency],
// eslint-disable-next-line no-undefined
undefined,
// eslint-disable-next-line no-undefined
undefined,
null,
(error, snapshot) => {
if (error) {
reject(error);

return;
}

resolve(snapshot);
}
);
});
}

static async checkSnapshotValid(compilation, snapshot) {
if (!compilation.fileSystemInfo) {
return;
}

// eslint-disable-next-line consistent-return
return new Promise((resolve, reject) => {
compilation.fileSystemInfo.checkSnapshotValid(
snapshot,
(error, isValid) => {
if (error) {
reject(error);

return;
}

resolve(isValid);
}
);
});
}

// eslint-disable-next-line class-methods-use-this
async runPattern(compiler, compilation, logger, inputPattern) {
static async runPattern(compiler, compilation, logger, cache, inputPattern) {
const pattern =
typeof inputPattern === 'string'
? { from: inputPattern }
Expand All @@ -49,7 +99,6 @@ class CopyPlugin {
pattern.to = path.normalize(
typeof pattern.to !== 'undefined' ? pattern.to : ''
);

pattern.context = path.normalize(
typeof pattern.context !== 'undefined'
? !path.isAbsolute(pattern.context)
Expand Down Expand Up @@ -266,64 +315,109 @@ class CopyPlugin {
compilation.fileDependencies.add(file.absoluteFrom);
}

logger.debug(`reading "${file.absoluteFrom}" to write to assets`);
let source;

let data;
if (cache) {
const cacheEntry = await cache.getPromise(file.relativeFrom, null);

try {
data = await readFile(inputFileSystem, file.absoluteFrom);
} catch (error) {
compilation.errors.push(error);
if (cacheEntry) {
const isValidSnapshot = await CopyPlugin.checkSnapshotValid(
compilation,
cacheEntry.snapshot
);

return;
if (isValidSnapshot) {
({ source } = cacheEntry);
}
}
}

if (pattern.transform) {
logger.log(`transforming content for "${file.absoluteFrom}"`);

if (pattern.cacheTransform) {
const cacheDirectory = pattern.cacheTransform.directory
? pattern.cacheTransform.directory
: typeof pattern.cacheTransform === 'string'
? pattern.cacheTransform
: findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
let defaultCacheKeys = {
version,
transform: pattern.transform,
contentHash: crypto.createHash('md4').update(data).digest('hex'),
};

if (typeof pattern.cacheTransform.keys === 'function') {
defaultCacheKeys = await pattern.cacheTransform.keys(
defaultCacheKeys,
file.absoluteFrom
);
} else {
defaultCacheKeys = {
...defaultCacheKeys,
...pattern.cacheTransform.keys,
if (!source) {
let startTime;

if (cache) {
startTime = Date.now();
}

logger.debug(`reading "${file.absoluteFrom}" to write to assets`);

let data;

try {
data = await readFile(inputFileSystem, file.absoluteFrom);
} catch (error) {
compilation.errors.push(error);

return;
}

if (pattern.transform) {
logger.log(`transforming content for "${file.absoluteFrom}"`);

if (pattern.cacheTransform) {
const cacheDirectory = pattern.cacheTransform.directory
? pattern.cacheTransform.directory
: typeof pattern.cacheTransform === 'string'
? pattern.cacheTransform
: findCacheDir({ name: 'copy-webpack-plugin' }) || os.tmpdir();
let defaultCacheKeys = {
version,
transform: pattern.transform,
contentHash: crypto
.createHash('md4')
.update(data)
.digest('hex'),
};
}

const cacheKeys = serialize(defaultCacheKeys);
if (typeof pattern.cacheTransform.keys === 'function') {
defaultCacheKeys = await pattern.cacheTransform.keys(
defaultCacheKeys,
file.absoluteFrom
);
} else {
defaultCacheKeys = {
...defaultCacheKeys,
...pattern.cacheTransform.keys,
};
}

try {
const result = await cacache.get(cacheDirectory, cacheKeys);
const cacheKeys = serialize(defaultCacheKeys);

logger.debug(
`getting cached transformation for "${file.absoluteFrom}"`
);
try {
const result = await cacache.get(cacheDirectory, cacheKeys);

({ data } = result);
} catch (_ignoreError) {
data = await pattern.transform(data, file.absoluteFrom);
logger.debug(
`getting cached transformation for "${file.absoluteFrom}"`
);

logger.debug(`caching transformation for "${file.absoluteFrom}"`);
({ data } = result);
} catch (_ignoreError) {
data = await pattern.transform(data, file.absoluteFrom);

logger.debug(
`caching transformation for "${file.absoluteFrom}"`
);

await cacache.put(cacheDirectory, cacheKeys, data);
await cacache.put(cacheDirectory, cacheKeys, data);
}
} else {
data = await pattern.transform(data, file.absoluteFrom);
}
} else {
data = await pattern.transform(data, file.absoluteFrom);
}

source = new RawSource(data);

if (cache) {
const snapshot = await CopyPlugin.createSnapshot(
compilation,
startTime,
file.relativeFrom
);

await cache.storePromise(file.relativeFrom, null, {
source,
snapshot,
});
}
}

Expand All @@ -349,7 +443,7 @@ class CopyPlugin {
{ resourcePath: file.absoluteFrom },
file.webpackTo,
{
content: data,
content: source.source(),
context: pattern.context,
}
);
Expand All @@ -374,7 +468,7 @@ class CopyPlugin {
}

// eslint-disable-next-line no-param-reassign
file.data = data;
file.source = source;
// eslint-disable-next-line no-param-reassign
file.targetPath = normalizePath(file.webpackTo);
// eslint-disable-next-line no-param-reassign
Expand All @@ -392,6 +486,10 @@ class CopyPlugin {

compiler.hooks.thisCompilation.tap(pluginName, (compilation) => {
const logger = compilation.getLogger('copy-webpack-plugin');
const cache = compilation.getCache
? compilation.getCache('CopyWebpackPlugin')
: // eslint-disable-next-line no-undefined
undefined;

compilation.hooks.additionalAssets.tapAsync(
'copy-webpack-plugin',
Expand All @@ -404,7 +502,13 @@ class CopyPlugin {
assets = await Promise.all(
this.patterns.map((item) =>
limit(async () =>
this.runPattern(compiler, compilation, logger, item)
CopyPlugin.runPattern(
compiler,
compilation,
logger,
cache,
item
)
)
)
);
Expand All @@ -426,12 +530,10 @@ class CopyPlugin {
absoluteFrom,
targetPath,
webpackTo,
data,
source,
force,
} = asset;

const source = new RawSource(data);

// For old version webpack 4
/* istanbul ignore if */
if (typeof compilation.emitAsset !== 'function') {
Expand Down
70 changes: 67 additions & 3 deletions test/CopyPlugin.test.js
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
import path from 'path';

import webpack from 'webpack';
import del from 'del';

import CopyPlugin from '../src';

Expand Down Expand Up @@ -633,9 +634,72 @@ describe('CopyPlugin', () => {
.then(done)
.catch(done);
});
});

describe('cache', () => {
it('should work with the "memory" cache', async () => {
const compiler = getCompiler({
cache: {
type: 'memory',
},
});

new CopyPlugin({
patterns: [
{
from: path.resolve(__dirname, './fixtures/directory'),
},
],
}).apply(compiler);

const { stats } = await compile(compiler);

if (webpack.version[0] === '4') {
expect(
Object.keys(stats.compilation.assets).filter(
(assetName) => stats.compilation.assets[assetName].emitted
).length
).toBe(5);
} else {
expect(stats.compilation.emittedAssets.size).toBe(5);
}

expect(readAssets(compiler, stats)).toMatchSnapshot('assets');
expect(stats.compilation.errors).toMatchSnapshot('errors');
expect(stats.compilation.warnings).toMatchSnapshot('warnings');

await new Promise(async (resolve) => {
const { stats: newStats } = await compile(compiler);

if (webpack.version[0] === '4') {
expect(
Object.keys(newStats.compilation.assets).filter(
(assetName) => newStats.compilation.assets[assetName].emitted
).length
).toBe(4);
} else {
expect(newStats.compilation.emittedAssets.size).toBe(0);
}

it('should work and do not emit unchanged assets', async () => {
const compiler = getCompiler();
expect(readAssets(compiler, newStats)).toMatchSnapshot('assets');
expect(newStats.compilation.errors).toMatchSnapshot('errors');
expect(newStats.compilation.warnings).toMatchSnapshot('warnings');

resolve();
});
});

it('should work with the "filesystem" cache', async () => {
const cacheDirectory = path.resolve(__dirname, './outputs/.cache');

await del(cacheDirectory);

const compiler = getCompiler({
cache: {
type: 'filesystem',
cacheDirectory,
},
});

new CopyPlugin({
patterns: [
Expand Down Expand Up @@ -671,7 +735,7 @@ describe('CopyPlugin', () => {
).length
).toBe(4);
} else {
expect(newStats.compilation.emittedAssets.size).toBe(4);
expect(newStats.compilation.emittedAssets.size).toBe(0);
}

expect(readAssets(compiler, newStats)).toMatchSnapshot('assets');
Expand Down

0 comments on commit c892451

Please sign in to comment.