From f809c7971d458f947171cf085dfe24e810cc9ce4 Mon Sep 17 00:00:00 2001 From: scotthovestadt Date: Wed, 20 Mar 2019 16:02:31 -0700 Subject: [PATCH] Optimize haste map data structure for serialization/deserialization. (#8171) * Optimize haste map data structure for serialization/deserialization. * Missing files. * Wrong delim in test file. * Update CHANGELOG.md * Fix crawler tests. * Switch delim to `:` and extract to var. * Switch delim to \0. * Fix dependency delim type. * Remove unnecessary cache file version. * Do not return empty element in dependency array. --- CHANGELOG.md | 1 + packages/jest-haste-map/src/HasteFS.ts | 9 ++- .../src/__tests__/index.test.js | 64 ++++++++----------- packages/jest-haste-map/src/constants.ts | 3 + .../src/crawlers/__tests__/node.test.js | 32 +++++----- .../src/crawlers/__tests__/watchman.test.js | 34 +++++----- packages/jest-haste-map/src/crawlers/node.ts | 2 +- .../jest-haste-map/src/crawlers/watchman.ts | 2 +- packages/jest-haste-map/src/index.ts | 6 +- packages/jest-haste-map/src/types.ts | 3 +- 10 files changed, 78 insertions(+), 78 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae7b2e733a41..83864ba1d072 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -32,6 +32,7 @@ ### Performance +- `[jest-haste-map]` Optimize haste map data structure for serialization/deserialization ([#8171](https://github.com/facebook/jest/pull/8171)) - `[jest-haste-map]` Avoid persisting haste map or processing files when not changed ([#8153](https://github.com/facebook/jest/pull/8153)) ## 24.5.0 diff --git a/packages/jest-haste-map/src/HasteFS.ts b/packages/jest-haste-map/src/HasteFS.ts index de3628871f27..0007da2d6ae9 100644 --- a/packages/jest-haste-map/src/HasteFS.ts +++ b/packages/jest-haste-map/src/HasteFS.ts @@ -33,7 +33,14 @@ export default class HasteFS { getDependencies(file: Config.Path): Array | null { const fileMetadata = this._getFileData(file); - return (fileMetadata && fileMetadata[H.DEPENDENCIES]) || null; + + if (fileMetadata) { + return fileMetadata[H.DEPENDENCIES] + ? fileMetadata[H.DEPENDENCIES].split(H.DEPENDENCY_DELIM) + : []; + } else { + return null; + } } getSha1(file: Config.Path): string | null { diff --git a/packages/jest-haste-map/src/__tests__/index.test.js b/packages/jest-haste-map/src/__tests__/index.test.js index 03b74317a210..a24f9c3bddbf 100644 --- a/packages/jest-haste-map/src/__tests__/index.test.js +++ b/packages/jest-haste-map/src/__tests__/index.test.js @@ -340,17 +340,17 @@ describe('HasteMap', () => { expect(data.files).toEqual( createMap({ - 'fruits/Banana.js': ['Banana', 32, 42, 1, ['Strawberry'], null], - 'fruits/Pear.js': ['Pear', 32, 42, 1, ['Banana', 'Strawberry'], null], - 'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, [], null], - 'fruits/__mocks__/Pear.js': ['', 32, 42, 1, ['Melon'], null], + 'fruits/Banana.js': ['Banana', 32, 42, 1, 'Strawberry', null], + 'fruits/Pear.js': ['Pear', 32, 42, 1, 'Banana\0Strawberry', null], + 'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, '', null], + 'fruits/__mocks__/Pear.js': ['', 32, 42, 1, 'Melon', null], // node modules 'fruits/node_modules/fbjs/lib/flatMap.js': [ 'flatMap', 32, 42, 1, - [], + '', null, ], 'fruits/node_modules/react/React.js': [ @@ -358,10 +358,10 @@ describe('HasteMap', () => { 32, 42, 1, - ['Component'], + 'Component', null, ], - 'vegetables/Melon.js': ['Melon', 32, 42, 1, [], null], + 'vegetables/Melon.js': ['Melon', 32, 42, 1, '', null], }), ); @@ -410,18 +410,11 @@ describe('HasteMap', () => { // The node crawler returns "null" for the SHA-1. data.files = createMap({ - 'fruits/Banana.js': ['Banana', 32, 42, 0, ['Strawberry'], null], - 'fruits/Pear.js': [ - 'Pear', - 32, - 42, - 0, - ['Banana', 'Strawberry'], - null, - ], - 'fruits/Strawberry.js': ['Strawberry', 32, 42, 0, [], null], - 'fruits/__mocks__/Pear.js': ['', 32, 42, 0, ['Melon'], null], - 'vegetables/Melon.js': ['Melon', 32, 42, 0, [], null], + 'fruits/Banana.js': ['Banana', 32, 42, 0, 'Strawberry', null], + 'fruits/Pear.js': ['Pear', 32, 42, 0, 'Banana\0Strawberry', null], + 'fruits/Strawberry.js': ['Strawberry', 32, 42, 0, '', null], + 'fruits/__mocks__/Pear.js': ['', 32, 42, 0, 'Melon', null], + 'vegetables/Melon.js': ['Melon', 32, 42, 0, '', null], }); return Promise.resolve({ @@ -446,7 +439,7 @@ describe('HasteMap', () => { 32, 42, 1, - ['Strawberry'], + 'Strawberry', '7772b628e422e8cf59c526be4bb9f44c0898e3d1', ], 'fruits/Pear.js': [ @@ -454,7 +447,7 @@ describe('HasteMap', () => { 32, 42, 1, - ['Banana', 'Strawberry'], + 'Banana\0Strawberry', '89d0c2cc11dcc5e1df50b8af04ab1b597acfba2f', ], 'fruits/Strawberry.js': [ @@ -462,7 +455,7 @@ describe('HasteMap', () => { 32, 42, 1, - [], + '', 'e8aa38e232b3795f062f1d777731d9240c0f8c25', ], 'fruits/__mocks__/Pear.js': [ @@ -470,7 +463,7 @@ describe('HasteMap', () => { 32, 42, 1, - ['Melon'], + 'Melon', '8d40afbb6e2dc78e1ba383b6d02cafad35cceef2', ], 'vegetables/Melon.js': [ @@ -478,7 +471,7 @@ describe('HasteMap', () => { 32, 42, 1, - [], + '', 'f16ccf6f2334ceff2ddb47628a2c5f2d748198ca', ], }), @@ -626,7 +619,7 @@ describe('HasteMap', () => { 32, 42, 1, - ['Blackberry'], + 'Blackberry', null, ], 'fruits/Strawberry.ios.js': [ @@ -634,10 +627,10 @@ describe('HasteMap', () => { 32, 42, 1, - ['Raspberry'], + 'Raspberry', null, ], - 'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, ['Banana'], null], + 'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, 'Banana', null], }), ); @@ -724,14 +717,7 @@ describe('HasteMap', () => { expect(useBuitinsInContext(data.clocks)).toEqual(mockClocks); const files = new Map(initialData.files); - files.set('fruits/Banana.js', [ - 'Banana', - 32, - 42, - 1, - ['Kiwi'], - null, - ]); + files.set('fruits/Banana.js', ['Banana', 32, 42, 1, 'Kiwi', null]); expect(useBuitinsInContext(data.files)).toEqual(files); @@ -1086,7 +1072,7 @@ describe('HasteMap', () => { node.mockImplementation(options => { const {data} = options; data.files = createMap({ - 'fruits/Banana.js': ['', 32, 42, 0, [], null], + 'fruits/Banana.js': ['', 32, 42, 0, '', null], }); return Promise.resolve({ hasteMap: data, @@ -1102,7 +1088,7 @@ describe('HasteMap', () => { expect(data.files).toEqual( createMap({ - 'fruits/Banana.js': ['Banana', 32, 42, 1, ['Strawberry'], null], + 'fruits/Banana.js': ['Banana', 32, 42, 1, 'Strawberry', null], }), ); @@ -1120,7 +1106,7 @@ describe('HasteMap', () => { node.mockImplementation(options => { const {data} = options; data.files = createMap({ - 'fruits/Banana.js': ['', 32, 42, 0, [], null], + 'fruits/Banana.js': ['', 32, 42, 0, '', null], }); return Promise.resolve({ hasteMap: data, @@ -1136,7 +1122,7 @@ describe('HasteMap', () => { expect(data.files).toEqual( createMap({ - 'fruits/Banana.js': ['Banana', 32, 42, 1, ['Strawberry'], null], + 'fruits/Banana.js': ['Banana', 32, 42, 1, 'Strawberry', null], }), ); }); diff --git a/packages/jest-haste-map/src/constants.ts b/packages/jest-haste-map/src/constants.ts index 1717934e47ef..ae949fe3cfd2 100644 --- a/packages/jest-haste-map/src/constants.ts +++ b/packages/jest-haste-map/src/constants.ts @@ -18,6 +18,9 @@ import {HType} from './types'; const constants: HType = { + /* dependency serialization */ + DEPENDENCY_DELIM: '\0', + /* file map attributes */ ID: 0, MTIME: 1, diff --git a/packages/jest-haste-map/src/crawlers/__tests__/node.test.js b/packages/jest-haste-map/src/crawlers/__tests__/node.test.js index b1cb1f1730e2..69d3c3d6ca73 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/node.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/node.test.js @@ -133,9 +133,9 @@ describe('node crawler', () => { expect(hasteMap.files).toEqual( createMap({ - 'fruits/strawberry.js': ['', 32, 42, 0, [], null], - 'fruits/tomato.js': ['', 33, 42, 0, [], null], - 'vegetables/melon.json': ['', 34, 42, 0, [], null], + 'fruits/strawberry.js': ['', 32, 42, 0, '', null], + 'fruits/tomato.js': ['', 33, 42, 0, '', null], + 'vegetables/melon.json': ['', 34, 42, 0, '', null], }), ); @@ -151,9 +151,9 @@ describe('node crawler', () => { nodeCrawl = require('../node'); // In this test sample, strawberry is changed and tomato is unchanged - const tomato = ['', 33, 42, 1, [], null]; + const tomato = ['', 33, 42, 1, '', null]; const files = createMap({ - 'fruits/strawberry.js': ['', 30, 40, 1, [], null], + 'fruits/strawberry.js': ['', 30, 40, 1, '', null], 'fruits/tomato.js': tomato, }); @@ -166,7 +166,7 @@ describe('node crawler', () => { }).then(({hasteMap, removedFiles}) => { expect(hasteMap.files).toEqual( createMap({ - 'fruits/strawberry.js': ['', 32, 42, 0, [], null], + 'fruits/strawberry.js': ['', 32, 42, 0, '', null], 'fruits/tomato.js': tomato, }), ); @@ -186,9 +186,9 @@ describe('node crawler', () => { // In this test sample, previouslyExisted was present before and will not be // when crawling this directory. const files = createMap({ - 'fruits/previouslyExisted.js': ['', 30, 40, 1, [], null], - 'fruits/strawberry.js': ['', 33, 42, 0, [], null], - 'fruits/tomato.js': ['', 32, 42, 0, [], null], + 'fruits/previouslyExisted.js': ['', 30, 40, 1, '', null], + 'fruits/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], }); return nodeCrawl({ @@ -200,13 +200,13 @@ describe('node crawler', () => { }).then(({hasteMap, removedFiles}) => { expect(hasteMap.files).toEqual( createMap({ - 'fruits/strawberry.js': ['', 32, 42, 0, [], null], - 'fruits/tomato.js': ['', 33, 42, 0, [], null], + 'fruits/strawberry.js': ['', 32, 42, 0, '', null], + 'fruits/tomato.js': ['', 33, 42, 0, '', null], }), ); expect(removedFiles).toEqual( createMap({ - 'fruits/previouslyExisted.js': ['', 30, 40, 1, [], null], + 'fruits/previouslyExisted.js': ['', 30, 40, 1, '', null], }), ); }); @@ -228,8 +228,8 @@ describe('node crawler', () => { }).then(({hasteMap, removedFiles}) => { expect(hasteMap.files).toEqual( createMap({ - 'fruits/directory/strawberry.js': ['', 33, 42, 0, [], null], - 'fruits/tomato.js': ['', 32, 42, 0, [], null], + 'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], }), ); expect(removedFiles).toEqual(new Map()); @@ -252,8 +252,8 @@ describe('node crawler', () => { }).then(({hasteMap, removedFiles}) => { expect(hasteMap.files).toEqual( createMap({ - 'fruits/directory/strawberry.js': ['', 33, 42, 0, [], null], - 'fruits/tomato.js': ['', 32, 42, 0, [], null], + 'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null], + 'fruits/tomato.js': ['', 32, 42, 0, '', null], }), ); expect(removedFiles).toEqual(new Map()); diff --git a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js index 06f2d73543ec..f72dc3fc5acb 100644 --- a/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js +++ b/packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js @@ -108,9 +108,9 @@ describe('watchman watch', () => { }; mockFiles = createMap({ - [MELON_RELATIVE]: ['', 33, 43, 0, [], null], - [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null], - [TOMATO_RELATIVE]: ['', 31, 41, 0, [], null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], }); }); @@ -216,8 +216,8 @@ describe('watchman watch', () => { expect(changedFiles).toEqual(undefined); expect(hasteMap.files).toEqual( createMap({ - [path.join(DURIAN_RELATIVE, 'foo.1.js')]: ['', 33, 43, 0, [], null], - [path.join(DURIAN_RELATIVE, 'foo.2.js')]: ['', 33, 43, 0, [], null], + [path.join(DURIAN_RELATIVE, 'foo.1.js')]: ['', 33, 43, 0, '', null], + [path.join(DURIAN_RELATIVE, 'foo.2.js')]: ['', 33, 43, 0, '', null], }), ); expect(removedFiles).toEqual(new Map()); @@ -280,21 +280,21 @@ describe('watchman watch', () => { expect(changedFiles).toEqual( createMap({ - [KIWI_RELATIVE]: ['', 42, 40, 0, [], null], + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], }), ); expect(hasteMap.files).toEqual( createMap({ - [KIWI_RELATIVE]: ['', 42, 40, 0, [], null], - [MELON_RELATIVE]: ['', 33, 43, 0, [], null], - [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null], + [KIWI_RELATIVE]: ['', 42, 40, 0, '', null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], }), ); expect(removedFiles).toEqual( createMap({ - [TOMATO_RELATIVE]: ['', 31, 41, 0, [], null], + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], }), ); }); @@ -374,7 +374,7 @@ describe('watchman watch', () => { expect(hasteMap.files).toEqual( createMap({ [BANANA_RELATIVE]: mockBananaMetadata, - [KIWI_RELATIVE]: ['', 42, 52, 0, [], null], + [KIWI_RELATIVE]: ['', 42, 52, 0, '', null], [TOMATO_RELATIVE]: ['Tomato', 76, 41, 1, [], mockTomatoSha1], }), ); @@ -388,8 +388,8 @@ describe('watchman watch', () => { expect(removedFiles).toEqual( createMap({ - [MELON_RELATIVE]: ['', 33, 43, 0, [], null], - [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], }), ); }); @@ -466,15 +466,15 @@ describe('watchman watch', () => { expect(hasteMap.files).toEqual( createMap({ - [KIWI_RELATIVE]: ['', 42, 52, 0, [], null], - [MELON_RELATIVE]: ['', 33, 43, 0, [], null], + [KIWI_RELATIVE]: ['', 42, 52, 0, '', null], + [MELON_RELATIVE]: ['', 33, 43, 0, '', null], }), ); expect(removedFiles).toEqual( createMap({ - [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null], - [TOMATO_RELATIVE]: ['', 31, 41, 0, [], null], + [STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null], + [TOMATO_RELATIVE]: ['', 31, 41, 0, '', null], }), ); }); diff --git a/packages/jest-haste-map/src/crawlers/node.ts b/packages/jest-haste-map/src/crawlers/node.ts index 07a93edf3ee4..71ee6c053a3b 100644 --- a/packages/jest-haste-map/src/crawlers/node.ts +++ b/packages/jest-haste-map/src/crawlers/node.ts @@ -164,7 +164,7 @@ export = function nodeCrawl( files.set(relativeFilePath, existingFile); } else { // See ../constants.js; SHA-1 will always be null and fulfilled later. - files.set(relativeFilePath, ['', mtime, size, 0, [], null]); + files.set(relativeFilePath, ['', mtime, size, 0, '', null]); } removedFiles.delete(relativeFilePath); }); diff --git a/packages/jest-haste-map/src/crawlers/watchman.ts b/packages/jest-haste-map/src/crawlers/watchman.ts index c6d6b66766d9..5e60ed59a7e4 100644 --- a/packages/jest-haste-map/src/crawlers/watchman.ts +++ b/packages/jest-haste-map/src/crawlers/watchman.ts @@ -232,7 +232,7 @@ export = async function watchmanCrawl( ]; } else { // See ../constants.ts - nextData = ['', mtime, size, 0, [], sha1hex]; + nextData = ['', mtime, size, 0, '', sha1hex]; } const mappings = options.mapper ? options.mapper(filePath) : null; diff --git a/packages/jest-haste-map/src/index.ts b/packages/jest-haste-map/src/index.ts index a8662c2611ac..d39be76252cf 100644 --- a/packages/jest-haste-map/src/index.ts +++ b/packages/jest-haste-map/src/index.ts @@ -522,7 +522,9 @@ class HasteMap extends EventEmitter { setModule(metadataId, metadataModule); } - fileMetadata[H.DEPENDENCIES] = metadata.dependencies || []; + fileMetadata[H.DEPENDENCIES] = metadata.dependencies + ? metadata.dependencies.join(H.DEPENDENCY_DELIM) + : ''; if (computeSha1) { fileMetadata[H.SHA1] = metadata.sha1; @@ -944,7 +946,7 @@ class HasteMap extends EventEmitter { stat ? stat.mtime.getTime() : -1, stat ? stat.size : 0, 0, - [], + '', null, ]; hasteMap.files.set(relativeFilePath, fileMetadata); diff --git a/packages/jest-haste-map/src/types.ts b/packages/jest-haste-map/src/types.ts index 59d2f8022330..291d5473ba98 100644 --- a/packages/jest-haste-map/src/types.ts +++ b/packages/jest-haste-map/src/types.ts @@ -51,7 +51,7 @@ export type FileMetaData = [ /* mtime */ number, /* size */ number, /* visited */ 0 | 1, - /* dependencies */ Array, + /* dependencies */ string, /* sha1 */ string | null | undefined ]; @@ -100,6 +100,7 @@ export type HType = { PACKAGE: 1; GENERIC_PLATFORM: 'g'; NATIVE_PLATFORM: 'native'; + DEPENDENCY_DELIM: '\0'; }; export type HTypeValue = HType[keyof HType];