Skip to content

Commit

Permalink
Optimize haste map data structure for serialization/deserialization. (#…
Browse files Browse the repository at this point in the history 
…8171)

* Optimize haste map data structure for serialization/deserialization.

* Missing files.

* Wrong delim in test file.

* Update CHANGELOG.md

* Fix crawler tests.

* Switch delim to `:` and extract to var.

* Switch delim to \0.

* Fix dependency delim type.

* Remove unnecessary cache file version.

* Do not return empty element in dependency array.
  • Loading branch information
@scotthovestadt @jeysal
scotthovestadt authored and jeysal committed Mar 20, 2019
1 parent 921edc0 commit f809c79
Show file tree
Hide file tree
Showing 10 changed files with 78 additions and 78 deletions.
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Expand Up @@ -32,6 +32,7 @@

### Performance

- `[jest-haste-map]` Optimize haste map data structure for serialization/deserialization ([#8171](https://github.com/facebook/jest/pull/8171))
- `[jest-haste-map]` Avoid persisting haste map or processing files when not changed ([#8153](https://github.com/facebook/jest/pull/8153))

## 24.5.0
Expand Down
9 changes: 8 additions & 1 deletion packages/jest-haste-map/src/HasteFS.ts
View file
Expand Up @@ -33,7 +33,14 @@ export default class HasteFS {

getDependencies(file: Config.Path): Array<string> | null {
const fileMetadata = this._getFileData(file);
return (fileMetadata && fileMetadata[H.DEPENDENCIES]) || null;

if (fileMetadata) {
return fileMetadata[H.DEPENDENCIES]
? fileMetadata[H.DEPENDENCIES].split(H.DEPENDENCY_DELIM)
: [];
} else {
return null;
}
}

getSha1(file: Config.Path): string | null {
Expand Down
64 changes: 25 additions & 39 deletions packages/jest-haste-map/src/__tests__/index.test.js
View file
Expand Up @@ -340,28 +340,28 @@ describe('HasteMap', () => {

expect(data.files).toEqual(
createMap({
'fruits/Banana.js': ['Banana', 32, 42, 1, ['Strawberry'], null],
'fruits/Pear.js': ['Pear', 32, 42, 1, ['Banana', 'Strawberry'], null],
'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, [], null],
'fruits/__mocks__/Pear.js': ['', 32, 42, 1, ['Melon'], null],
'fruits/Banana.js': ['Banana', 32, 42, 1, 'Strawberry', null],
'fruits/Pear.js': ['Pear', 32, 42, 1, 'Banana\0Strawberry', null],
'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, '', null],
'fruits/__mocks__/Pear.js': ['', 32, 42, 1, 'Melon', null],
// node modules
'fruits/node_modules/fbjs/lib/flatMap.js': [
'flatMap',
32,
42,
1,
[],
'',
null,
],
'fruits/node_modules/react/React.js': [
'React',
32,
42,
1,
['Component'],
'Component',
null,
],
'vegetables/Melon.js': ['Melon', 32, 42, 1, [], null],
'vegetables/Melon.js': ['Melon', 32, 42, 1, '', null],
}),
);

Expand Down Expand Up @@ -410,18 +410,11 @@ describe('HasteMap', () => {

// The node crawler returns "null" for the SHA-1.
data.files = createMap({
'fruits/Banana.js': ['Banana', 32, 42, 0, ['Strawberry'], null],
'fruits/Pear.js': [
'Pear',
32,
42,
0,
['Banana', 'Strawberry'],
null,
],
'fruits/Strawberry.js': ['Strawberry', 32, 42, 0, [], null],
'fruits/__mocks__/Pear.js': ['', 32, 42, 0, ['Melon'], null],
'vegetables/Melon.js': ['Melon', 32, 42, 0, [], null],
'fruits/Banana.js': ['Banana', 32, 42, 0, 'Strawberry', null],
'fruits/Pear.js': ['Pear', 32, 42, 0, 'Banana\0Strawberry', null],
'fruits/Strawberry.js': ['Strawberry', 32, 42, 0, '', null],
'fruits/__mocks__/Pear.js': ['', 32, 42, 0, 'Melon', null],
'vegetables/Melon.js': ['Melon', 32, 42, 0, '', null],
});

return Promise.resolve({
Expand All @@ -446,39 +439,39 @@ describe('HasteMap', () => {
32,
42,
1,
['Strawberry'],
'Strawberry',
'7772b628e422e8cf59c526be4bb9f44c0898e3d1',
],
'fruits/Pear.js': [
'Pear',
32,
42,
1,
['Banana', 'Strawberry'],
'Banana\0Strawberry',
'89d0c2cc11dcc5e1df50b8af04ab1b597acfba2f',
],
'fruits/Strawberry.js': [
'Strawberry',
32,
42,
1,
[],
'',
'e8aa38e232b3795f062f1d777731d9240c0f8c25',
],
'fruits/__mocks__/Pear.js': [
'',
32,
42,
1,
['Melon'],
'Melon',
'8d40afbb6e2dc78e1ba383b6d02cafad35cceef2',
],
'vegetables/Melon.js': [
'Melon',
32,
42,
1,
[],
'',
'f16ccf6f2334ceff2ddb47628a2c5f2d748198ca',
],
}),
Expand Down Expand Up @@ -626,18 +619,18 @@ describe('HasteMap', () => {
32,
42,
1,
['Blackberry'],
'Blackberry',
null,
],
'fruits/Strawberry.ios.js': [
'Strawberry',
32,
42,
1,
['Raspberry'],
'Raspberry',
null,
],
'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, ['Banana'], null],
'fruits/Strawberry.js': ['Strawberry', 32, 42, 1, 'Banana', null],
}),
);

Expand Down Expand Up @@ -724,14 +717,7 @@ describe('HasteMap', () => {
expect(useBuitinsInContext(data.clocks)).toEqual(mockClocks);

const files = new Map(initialData.files);
files.set('fruits/Banana.js', [
'Banana',
32,
42,
1,
['Kiwi'],
null,
]);
files.set('fruits/Banana.js', ['Banana', 32, 42, 1, 'Kiwi', null]);

expect(useBuitinsInContext(data.files)).toEqual(files);

Expand Down Expand Up @@ -1086,7 +1072,7 @@ describe('HasteMap', () => {
node.mockImplementation(options => {
const {data} = options;
data.files = createMap({
'fruits/Banana.js': ['', 32, 42, 0, [], null],
'fruits/Banana.js': ['', 32, 42, 0, '', null],
});
return Promise.resolve({
hasteMap: data,
Expand All @@ -1102,7 +1088,7 @@ describe('HasteMap', () => {

expect(data.files).toEqual(
createMap({
'fruits/Banana.js': ['Banana', 32, 42, 1, ['Strawberry'], null],
'fruits/Banana.js': ['Banana', 32, 42, 1, 'Strawberry', null],
}),
);

Expand All @@ -1120,7 +1106,7 @@ describe('HasteMap', () => {
node.mockImplementation(options => {
const {data} = options;
data.files = createMap({
'fruits/Banana.js': ['', 32, 42, 0, [], null],
'fruits/Banana.js': ['', 32, 42, 0, '', null],
});
return Promise.resolve({
hasteMap: data,
Expand All @@ -1136,7 +1122,7 @@ describe('HasteMap', () => {

expect(data.files).toEqual(
createMap({
'fruits/Banana.js': ['Banana', 32, 42, 1, ['Strawberry'], null],
'fruits/Banana.js': ['Banana', 32, 42, 1, 'Strawberry', null],
}),
);
});
Expand Down
3 changes: 3 additions & 0 deletions packages/jest-haste-map/src/constants.ts
View file
Expand Up @@ -18,6 +18,9 @@
import {HType} from './types';

const constants: HType = {
/* dependency serialization */
DEPENDENCY_DELIM: '\0',

/* file map attributes */
ID: 0,
MTIME: 1,
Expand Down
32 changes: 16 additions & 16 deletions packages/jest-haste-map/src/crawlers/__tests__/node.test.js
View file
Expand Up @@ -133,9 +133,9 @@ describe('node crawler', () => {

expect(hasteMap.files).toEqual(
createMap({
'fruits/strawberry.js': ['', 32, 42, 0, [], null],
'fruits/tomato.js': ['', 33, 42, 0, [], null],
'vegetables/melon.json': ['', 34, 42, 0, [], null],
'fruits/strawberry.js': ['', 32, 42, 0, '', null],
'fruits/tomato.js': ['', 33, 42, 0, '', null],
'vegetables/melon.json': ['', 34, 42, 0, '', null],
}),
);

Expand All @@ -151,9 +151,9 @@ describe('node crawler', () => {
nodeCrawl = require('../node');

// In this test sample, strawberry is changed and tomato is unchanged
const tomato = ['', 33, 42, 1, [], null];
const tomato = ['', 33, 42, 1, '', null];
const files = createMap({
'fruits/strawberry.js': ['', 30, 40, 1, [], null],
'fruits/strawberry.js': ['', 30, 40, 1, '', null],
'fruits/tomato.js': tomato,
});

Expand All @@ -166,7 +166,7 @@ describe('node crawler', () => {
}).then(({hasteMap, removedFiles}) => {
expect(hasteMap.files).toEqual(
createMap({
'fruits/strawberry.js': ['', 32, 42, 0, [], null],
'fruits/strawberry.js': ['', 32, 42, 0, '', null],
'fruits/tomato.js': tomato,
}),
);
Expand All @@ -186,9 +186,9 @@ describe('node crawler', () => {
// In this test sample, previouslyExisted was present before and will not be
// when crawling this directory.
const files = createMap({
'fruits/previouslyExisted.js': ['', 30, 40, 1, [], null],
'fruits/strawberry.js': ['', 33, 42, 0, [], null],
'fruits/tomato.js': ['', 32, 42, 0, [], null],
'fruits/previouslyExisted.js': ['', 30, 40, 1, '', null],
'fruits/strawberry.js': ['', 33, 42, 0, '', null],
'fruits/tomato.js': ['', 32, 42, 0, '', null],
});

return nodeCrawl({
Expand All @@ -200,13 +200,13 @@ describe('node crawler', () => {
}).then(({hasteMap, removedFiles}) => {
expect(hasteMap.files).toEqual(
createMap({
'fruits/strawberry.js': ['', 32, 42, 0, [], null],
'fruits/tomato.js': ['', 33, 42, 0, [], null],
'fruits/strawberry.js': ['', 32, 42, 0, '', null],
'fruits/tomato.js': ['', 33, 42, 0, '', null],
}),
);
expect(removedFiles).toEqual(
createMap({
'fruits/previouslyExisted.js': ['', 30, 40, 1, [], null],
'fruits/previouslyExisted.js': ['', 30, 40, 1, '', null],
}),
);
});
Expand All @@ -228,8 +228,8 @@ describe('node crawler', () => {
}).then(({hasteMap, removedFiles}) => {
expect(hasteMap.files).toEqual(
createMap({
'fruits/directory/strawberry.js': ['', 33, 42, 0, [], null],
'fruits/tomato.js': ['', 32, 42, 0, [], null],
'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null],
'fruits/tomato.js': ['', 32, 42, 0, '', null],
}),
);
expect(removedFiles).toEqual(new Map());
Expand All @@ -252,8 +252,8 @@ describe('node crawler', () => {
}).then(({hasteMap, removedFiles}) => {
expect(hasteMap.files).toEqual(
createMap({
'fruits/directory/strawberry.js': ['', 33, 42, 0, [], null],
'fruits/tomato.js': ['', 32, 42, 0, [], null],
'fruits/directory/strawberry.js': ['', 33, 42, 0, '', null],
'fruits/tomato.js': ['', 32, 42, 0, '', null],
}),
);
expect(removedFiles).toEqual(new Map());
Expand Down
34 changes: 17 additions & 17 deletions packages/jest-haste-map/src/crawlers/__tests__/watchman.test.js
View file
Expand Up @@ -108,9 +108,9 @@ describe('watchman watch', () => {
};

mockFiles = createMap({
[MELON_RELATIVE]: ['', 33, 43, 0, [], null],
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null],
[TOMATO_RELATIVE]: ['', 31, 41, 0, [], null],
[MELON_RELATIVE]: ['', 33, 43, 0, '', null],
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null],
[TOMATO_RELATIVE]: ['', 31, 41, 0, '', null],
});
});

Expand Down Expand Up @@ -216,8 +216,8 @@ describe('watchman watch', () => {
expect(changedFiles).toEqual(undefined);
expect(hasteMap.files).toEqual(
createMap({
[path.join(DURIAN_RELATIVE, 'foo.1.js')]: ['', 33, 43, 0, [], null],
[path.join(DURIAN_RELATIVE, 'foo.2.js')]: ['', 33, 43, 0, [], null],
[path.join(DURIAN_RELATIVE, 'foo.1.js')]: ['', 33, 43, 0, '', null],
[path.join(DURIAN_RELATIVE, 'foo.2.js')]: ['', 33, 43, 0, '', null],
}),
);
expect(removedFiles).toEqual(new Map());
Expand Down Expand Up @@ -280,21 +280,21 @@ describe('watchman watch', () => {

expect(changedFiles).toEqual(
createMap({
[KIWI_RELATIVE]: ['', 42, 40, 0, [], null],
[KIWI_RELATIVE]: ['', 42, 40, 0, '', null],
}),
);

expect(hasteMap.files).toEqual(
createMap({
[KIWI_RELATIVE]: ['', 42, 40, 0, [], null],
[MELON_RELATIVE]: ['', 33, 43, 0, [], null],
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null],
[KIWI_RELATIVE]: ['', 42, 40, 0, '', null],
[MELON_RELATIVE]: ['', 33, 43, 0, '', null],
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null],
}),
);

expect(removedFiles).toEqual(
createMap({
[TOMATO_RELATIVE]: ['', 31, 41, 0, [], null],
[TOMATO_RELATIVE]: ['', 31, 41, 0, '', null],
}),
);
});
Expand Down Expand Up @@ -374,7 +374,7 @@ describe('watchman watch', () => {
expect(hasteMap.files).toEqual(
createMap({
[BANANA_RELATIVE]: mockBananaMetadata,
[KIWI_RELATIVE]: ['', 42, 52, 0, [], null],
[KIWI_RELATIVE]: ['', 42, 52, 0, '', null],
[TOMATO_RELATIVE]: ['Tomato', 76, 41, 1, [], mockTomatoSha1],
}),
);
Expand All @@ -388,8 +388,8 @@ describe('watchman watch', () => {

expect(removedFiles).toEqual(
createMap({
[MELON_RELATIVE]: ['', 33, 43, 0, [], null],
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null],
[MELON_RELATIVE]: ['', 33, 43, 0, '', null],
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null],
}),
);
});
Expand Down Expand Up @@ -466,15 +466,15 @@ describe('watchman watch', () => {

expect(hasteMap.files).toEqual(
createMap({
[KIWI_RELATIVE]: ['', 42, 52, 0, [], null],
[MELON_RELATIVE]: ['', 33, 43, 0, [], null],
[KIWI_RELATIVE]: ['', 42, 52, 0, '', null],
[MELON_RELATIVE]: ['', 33, 43, 0, '', null],
}),
);

expect(removedFiles).toEqual(
createMap({
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, [], null],
[TOMATO_RELATIVE]: ['', 31, 41, 0, [], null],
[STRAWBERRY_RELATIVE]: ['', 30, 40, 0, '', null],
[TOMATO_RELATIVE]: ['', 31, 41, 0, '', null],
}),
);
});
Expand Down
2 changes: 1 addition & 1 deletion packages/jest-haste-map/src/crawlers/node.ts
View file
Expand Up @@ -164,7 +164,7 @@ export = function nodeCrawl(
files.set(relativeFilePath, existingFile);
} else {
// See ../constants.js; SHA-1 will always be null and fulfilled later.
files.set(relativeFilePath, ['', mtime, size, 0, [], null]);
files.set(relativeFilePath, ['', mtime, size, 0, '', null]);
}
removedFiles.delete(relativeFilePath);
});
Expand Down

0 comments on commit f809c79

Please sign in to comment.