From 504cacee18b844a974e76e5c45b2568bf023d3ac Mon Sep 17 00:00:00 2001 From: Joe Lencioni Date: Tue, 23 Jun 2020 06:10:12 -0500 Subject: [PATCH] Improve Jest startup time and test runtime, particularly when running with coverage, by caching micromatch and avoiding recreating RegExp instances (#10131) Co-authored-by: Christoph Nakazawa --- CHANGELOG.md | 2 + packages/jest-core/src/SearchSource.ts | 19 ++-- .../src/__tests__/SearchSource.test.ts | 28 ++++++ packages/jest-haste-map/src/HasteFS.ts | 7 +- .../jest-transform/src/shouldInstrument.ts | 31 ++++-- packages/jest-util/package.json | 4 +- .../src/__tests__/globsToMatcher.test.ts | 72 ++++++++++++++ packages/jest-util/src/globsToMatcher.ts | 99 +++++++++++++++++++ packages/jest-util/src/index.ts | 1 + 9 files changed, 244 insertions(+), 19 deletions(-) create mode 100644 packages/jest-util/src/__tests__/globsToMatcher.test.ts create mode 100644 packages/jest-util/src/globsToMatcher.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index 9f7103b29bf0..bb662811e451 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -24,6 +24,8 @@ ### Performance +- `[jest-core, jest-transform, jest-haste-map]` Improve Jest startup time and test runtime, particularly when running with coverage, by caching micromatch and avoiding recreating RegExp instances ([#10131](https://github.com/facebook/jest/pull/10131)) + ## 26.0.1 ### Fixes diff --git a/packages/jest-core/src/SearchSource.ts b/packages/jest-core/src/SearchSource.ts index 48c630f2adb4..333d8539f314 100644 --- a/packages/jest-core/src/SearchSource.ts +++ b/packages/jest-core/src/SearchSource.ts @@ -16,7 +16,7 @@ import DependencyResolver = require('jest-resolve-dependencies'); import {escapePathForRegex} from 'jest-regex-util'; import {replaceRootDirInPath} from 'jest-config'; import {buildSnapshotResolver} from 'jest-snapshot'; -import {replacePathSepForGlob, testPathPatternToRegExp} from 'jest-util'; +import {globsToMatcher, testPathPatternToRegExp} from 'jest-util'; import type {Filter, Stats, TestPathCases} from './types'; export type SearchResult = { @@ -37,12 +37,19 @@ export type TestSelectionConfig = { watch?: boolean; }; -const globsToMatcher = (globs: Array) => (path: Config.Path) => - micromatch([replacePathSepForGlob(path)], globs, {dot: true}).length > 0; +const regexToMatcher = (testRegex: Config.ProjectConfig['testRegex']) => { + const regexes = testRegex.map(testRegex => new RegExp(testRegex)); -const regexToMatcher = (testRegex: Config.ProjectConfig['testRegex']) => ( - path: Config.Path, -) => testRegex.some(testRegex => new RegExp(testRegex).test(path)); + return (path: Config.Path) => + regexes.some(regex => { + const result = regex.test(path); + + // prevent stateful regexes from breaking, just in case + regex.lastIndex = 0; + + return result; + }); +}; const toTests = (context: Context, tests: Array) => tests.map(path => ({ diff --git a/packages/jest-core/src/__tests__/SearchSource.test.ts b/packages/jest-core/src/__tests__/SearchSource.test.ts index fbc4513ef1f6..5bb1deedc263 100644 --- a/packages/jest-core/src/__tests__/SearchSource.test.ts +++ b/packages/jest-core/src/__tests__/SearchSource.test.ts @@ -206,6 +206,34 @@ describe('SearchSource', () => { }); }); + it('finds tests matching a JS with overriding glob patterns', () => { + const {options: config} = normalize( + { + moduleFileExtensions: ['js', 'jsx'], + name, + rootDir, + testMatch: [ + '**/*.js?(x)', + '!**/test.js?(x)', + '**/test.js', + '!**/test.js', + ], + testRegex: '', + }, + {} as Config.Argv, + ); + + return findMatchingTests(config).then(data => { + const relPaths = toPaths(data.tests).map(absPath => + path.relative(rootDir, absPath), + ); + expect(relPaths.sort()).toEqual([ + path.normalize('module.jsx'), + path.normalize('no_tests.js'), + ]); + }); + }); + it('finds tests with default file extensions using testRegex', () => { const {options: config} = normalize( { diff --git a/packages/jest-haste-map/src/HasteFS.ts b/packages/jest-haste-map/src/HasteFS.ts index 358eee2e3676..bdad6f292a82 100644 --- a/packages/jest-haste-map/src/HasteFS.ts +++ b/packages/jest-haste-map/src/HasteFS.ts @@ -5,8 +5,7 @@ * LICENSE file in the root directory of this source tree. */ -import micromatch = require('micromatch'); -import {replacePathSepForGlob} from 'jest-util'; +import {globsToMatcher, replacePathSepForGlob} from 'jest-util'; import type {Config} from '@jest/types'; import type {FileData} from './types'; import * as fastPath from './lib/fast_path'; @@ -84,9 +83,11 @@ export default class HasteFS { root: Config.Path | null, ): Set { const files = new Set(); + const matcher = globsToMatcher(globs); + for (const file of this.getAbsoluteFileIterator()) { const filePath = root ? fastPath.relative(root, file) : file; - if (micromatch([replacePathSepForGlob(filePath)], globs).length > 0) { + if (matcher(replacePathSepForGlob(filePath))) { files.add(file); } } diff --git a/packages/jest-transform/src/shouldInstrument.ts b/packages/jest-transform/src/shouldInstrument.ts index d9fcdcd268c3..8b729ace3afb 100644 --- a/packages/jest-transform/src/shouldInstrument.ts +++ b/packages/jest-transform/src/shouldInstrument.ts @@ -8,7 +8,7 @@ import * as path from 'path'; import type {Config} from '@jest/types'; import {escapePathForRegex} from 'jest-regex-util'; -import {replacePathSepForGlob} from 'jest-util'; +import {globsToMatcher, replacePathSepForGlob} from 'jest-util'; import micromatch = require('micromatch'); import type {ShouldInstrumentOptions} from './types'; @@ -16,6 +16,20 @@ const MOCKS_PATTERN = new RegExp( escapePathForRegex(path.sep + '__mocks__' + path.sep), ); +const cachedRegexes = new Map(); +const getRegex = (regexStr: string) => { + if (!cachedRegexes.has(regexStr)) { + cachedRegexes.set(regexStr, new RegExp(regexStr)); + } + + const regex = cachedRegexes.get(regexStr)!; + + // prevent stateful regexes from breaking, just in case + regex.lastIndex = 0; + + return regex; +}; + export default function shouldInstrument( filename: Config.Path, options: ShouldInstrumentOptions, @@ -33,15 +47,15 @@ export default function shouldInstrument( } if ( - !config.testPathIgnorePatterns.some(pattern => !!filename.match(pattern)) + !config.testPathIgnorePatterns.some(pattern => + getRegex(pattern).test(filename), + ) ) { if (config.testRegex.some(regex => new RegExp(regex).test(filename))) { return false; } - if ( - micromatch([replacePathSepForGlob(filename)], config.testMatch).length - ) { + if (globsToMatcher(config.testMatch)(replacePathSepForGlob(filename))) { return false; } } @@ -59,10 +73,9 @@ export default function shouldInstrument( // still cover if `only` is specified !options.collectCoverageOnlyFrom && options.collectCoverageFrom.length && - micromatch( - [replacePathSepForGlob(path.relative(config.rootDir, filename))], - options.collectCoverageFrom, - ).length === 0 + !globsToMatcher(options.collectCoverageFrom)( + replacePathSepForGlob(path.relative(config.rootDir, filename)), + ) ) { return false; } diff --git a/packages/jest-util/package.json b/packages/jest-util/package.json index 10cfaa9030df..dc891b89a98d 100644 --- a/packages/jest-util/package.json +++ b/packages/jest-util/package.json @@ -14,11 +14,13 @@ "chalk": "^4.0.0", "graceful-fs": "^4.2.4", "is-ci": "^2.0.0", - "make-dir": "^3.0.0" + "make-dir": "^3.0.0", + "micromatch": "^4.0.2" }, "devDependencies": { "@types/graceful-fs": "^4.1.2", "@types/is-ci": "^2.0.0", + "@types/micromatch": "^4.0.0", "@types/node": "*" }, "engines": { diff --git a/packages/jest-util/src/__tests__/globsToMatcher.test.ts b/packages/jest-util/src/__tests__/globsToMatcher.test.ts new file mode 100644 index 000000000000..f676d7ae0360 --- /dev/null +++ b/packages/jest-util/src/__tests__/globsToMatcher.test.ts @@ -0,0 +1,72 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import micromatch = require('micromatch'); +import globsToMatcher from '../globsToMatcher'; + +it('works like micromatch with only positive globs', () => { + const globs = ['**/*.test.js', '**/*.test.jsx']; + const matcher = globsToMatcher(globs); + + expect(matcher('some-module.js')).toBe( + micromatch(['some-module.js'], globs).length > 0, + ); + + expect(matcher('some-module.test.js')).toBe( + micromatch(['some-module.test.js'], globs).length > 0, + ); +}); + +it('works like micromatch with a mix of overlapping positive and negative globs', () => { + const globs = ['**/*.js', '!**/*.test.js', '**/*.test.js']; + const matcher = globsToMatcher(globs); + + expect(matcher('some-module.js')).toBe( + micromatch(['some-module.js'], globs).length > 0, + ); + + expect(matcher('some-module.test.js')).toBe( + micromatch(['some-module.test.js'], globs).length > 0, + ); + + const globs2 = ['**/*.js', '!**/*.test.js', '**/*.test.js', '!**/*.test.js']; + const matcher2 = globsToMatcher(globs2); + + expect(matcher2('some-module.js')).toBe( + micromatch(['some-module.js'], globs2).length > 0, + ); + + expect(matcher2('some-module.test.js')).toBe( + micromatch(['some-module.test.js'], globs2).length > 0, + ); +}); + +it('works like micromatch with only negative globs', () => { + const globs = ['!**/*.test.js', '!**/*.test.jsx']; + const matcher = globsToMatcher(globs); + + expect(matcher('some-module.js')).toBe( + micromatch(['some-module.js'], globs).length > 0, + ); + + expect(matcher('some-module.test.js')).toBe( + micromatch(['some-module.test.js'], globs).length > 0, + ); +}); + +it('works like micromatch with empty globs', () => { + const globs = []; + const matcher = globsToMatcher(globs); + + expect(matcher('some-module.js')).toBe( + micromatch(['some-module.js'], globs).length > 0, + ); + + expect(matcher('some-module.test.js')).toBe( + micromatch(['some-module.test.js'], globs).length > 0, + ); +}); diff --git a/packages/jest-util/src/globsToMatcher.ts b/packages/jest-util/src/globsToMatcher.ts new file mode 100644 index 000000000000..e100d42cfcbf --- /dev/null +++ b/packages/jest-util/src/globsToMatcher.ts @@ -0,0 +1,99 @@ +/** + * Copyright (c) Facebook, Inc. and its affiliates. All Rights Reserved. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +import micromatch = require('micromatch'); +import type {Config} from '@jest/types'; +import replacePathSepForGlob from './replacePathSepForGlob'; + +const globsToMatchersMap = new Map< + string, + { + isMatch: (str: string) => boolean; + negated: boolean; + } +>(); + +const micromatchOptions = {dot: true}; + +/** + * Converts a list of globs into a function that matches a path against the + * globs. + * + * Every time micromatch is called, it will parse the glob strings and turn + * them into regexp instances. Instead of calling micromatch repeatedly with + * the same globs, we can use this function which will build the micromatch + * matchers ahead of time and then have an optimized path for determining + * whether an individual path matches. + * + * This function is intended to match the behavior of `micromatch()`. + * + * @example + * const isMatch = globsToMatcher(['*.js', '!*.test.js']); + * isMatch('pizza.js'); // true + * isMatch('pizza.test.js'); // false + */ +export default function globsToMatcher( + globs: Array, +): (path: Config.Path) => boolean { + if (globs.length === 0) { + // Since there were no globs given, we can simply have a fast path here and + // return with a very simple function. + return (_: Config.Path): boolean => false; + } + + const matchers = globs.map(glob => { + if (!globsToMatchersMap.has(glob)) { + // Matchers that are negated have different behavior than matchers that + // are not negated, so we need to store this information ahead of time. + const {negated} = micromatch.scan(glob, micromatchOptions); + + const matcher = { + isMatch: micromatch.matcher(glob, micromatchOptions), + negated, + }; + + globsToMatchersMap.set(glob, matcher); + } + + return globsToMatchersMap.get(glob)!; + }); + + return (path: Config.Path): boolean => { + const replacedPath = replacePathSepForGlob(path); + let kept = undefined; + let negatives = 0; + + for (let i = 0; i < matchers.length; i++) { + const {isMatch, negated} = matchers[i]; + + if (negated) { + negatives++; + } + + const matched = isMatch(replacedPath); + + if (!matched && negated) { + // The path was not matched, and the matcher is a negated matcher, so we + // want to omit the path. This means that the negative matcher is + // filtering the path out. + kept = false; + } else if (matched && !negated) { + // The path was matched, and the matcher is not a negated matcher, so we + // want to keep the path. + kept = true; + } + } + + // If all of the globs were negative globs, then we want to include the path + // as long as it was not explicitly not kept. Otherwise only include + // the path if it was kept. This allows sets of globs that are all negated + // to allow some paths to be matched, while sets of globs that are mixed + // negated and non-negated to cause the negated matchers to only omit paths + // and not keep them. + return negatives === matchers.length ? kept !== false : !!kept; + }; +} diff --git a/packages/jest-util/src/index.ts b/packages/jest-util/src/index.ts index cfd00f7fd55d..40d200b1930d 100644 --- a/packages/jest-util/src/index.ts +++ b/packages/jest-util/src/index.ts @@ -18,6 +18,7 @@ export {default as convertDescriptorToString} from './convertDescriptorToString' import * as specialChars from './specialChars'; export {default as replacePathSepForGlob} from './replacePathSepForGlob'; export {default as testPathPatternToRegExp} from './testPathPatternToRegExp'; +export {default as globsToMatcher} from './globsToMatcher'; import * as preRunMessage from './preRunMessage'; export {default as pluralize} from './pluralize'; export {default as formatTime} from './formatTime';