Skip to content
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.

Commit ead50f7

Browse files
cpojercpojer
committedApr 19, 2016
More compact storage micro-optimization.
1 parent fbea74b commit ead50f7

File tree

9 files changed

+157
-178
lines changed

9 files changed

+157
-178
lines changed
 

‎packages/jest-haste-map/src/constants.js

+36
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
/**
2+
* Copyright (c) 2014-present, Facebook, Inc. All rights reserved.
3+
*
4+
* This source code is licensed under the BSD-style license found in the
5+
* LICENSE file in the root directory of this source tree. An additional grant
6+
* of patent rights can be found in the PATENTS file in the same directory.
7+
*/
8+
9+
/*
10+
* This file exports a set of constants that are used for Jest's haste map
11+
* serialization. On very large repositories, the haste map cache becomes very
12+
* large to the point where it is the largest overhead in starting up Jest.
13+
*
14+
* This constant key map allows to keep the map smaller without having to build
15+
* a custom serialization library.
16+
*/
17+
module.exports = {
18+
/* shared in file map and module map */
19+
ID: 0,
20+
21+
/* file map attributes */
22+
MTIME: 1,
23+
VISITED: 2,
24+
DEPENDENCIES: 3,
25+
26+
/* module map attributes */
27+
PATH: 1,
28+
TYPE: 2,
29+
30+
/* module types */
31+
MODULE: 0,
32+
PACKAGE: 1,
33+
34+
/* platforms */
35+
GENERIC_PLATFORM: 'g',
36+
};

‎packages/jest-haste-map/src/crawlers/node.js

+11-14
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,14 @@
99

1010
'use strict';
1111

12+
const H = require('../constants');
13+
1214
const fs = require('fs');
1315
const os = require('os');
1416
const path = require('path');
1517
const spawn = require('child_process').spawn;
1618

17-
function find(roots, extensions, ignorePattern, callback) {
19+
function find(roots, extensions, ignore, callback) {
1820
const result = [];
1921
let activeCalls = 0;
2022

@@ -28,7 +30,7 @@ function find(roots, extensions, ignorePattern, callback) {
2830
}
2931

3032
names.forEach(file => {
31-
if (ignorePattern.test(file)) {
33+
if (ignore(file)) {
3234
return;
3335
}
3436
activeCalls++;
@@ -61,7 +63,7 @@ function find(roots, extensions, ignorePattern, callback) {
6163
roots.forEach(search);
6264
}
6365

64-
function findNative(roots, extensions, ignorePattern, callback) {
66+
function findNative(roots, extensions, ignore, callback) {
6567
const args = [].concat(roots);
6668
args.push('-type', 'f');
6769
extensions.forEach((ext, index) => {
@@ -80,7 +82,7 @@ function findNative(roots, extensions, ignorePattern, callback) {
8082
child.stdout.on('close', code => {
8183
const lines = stdout.trim()
8284
.split('\n')
83-
.filter(x => !ignorePattern.test(x));
85+
.filter(x => !ignore(x));
8486
const result = [];
8587
let count = lines.length;
8688
lines.forEach(path => {
@@ -96,33 +98,28 @@ function findNative(roots, extensions, ignorePattern, callback) {
9698
});
9799
}
98100

99-
module.exports = function nodeCrawl(roots, extensions, ignorePattern, data) {
101+
module.exports = function nodeCrawl(roots, extensions, ignore, data) {
100102
return new Promise(resolve => {
101103
const callback = list => {
102104
const files = Object.create(null);
103105
list.forEach(fileData => {
104106
const name = fileData[0];
105107
const mtime = fileData[1];
106108
const existingFile = data.files[name];
107-
if (existingFile && existingFile.mtime === mtime) {
108-
//console.log('exists', name);
109+
if (existingFile && existingFile[H.MTIME] === mtime) {
109110
files[name] = existingFile;
110111
} else {
111-
//console.log('add', name);
112-
files[name] = {
113-
mtime,
114-
visited: false,
115-
};
112+
files[name] = [0, mtime, 0, []];
116113
}
117114
});
118115
data.files = files;
119116
resolve(data);
120117
};
121118

122119
if (os.platform() == 'win32') {
123-
find(roots, extensions, ignorePattern, callback);
120+
find(roots, extensions, ignore, callback);
124121
} else {
125-
findNative(roots, extensions, ignorePattern, callback);
122+
findNative(roots, extensions, ignore, callback);
126123
}
127124
});
128125
};

‎packages/jest-haste-map/src/crawlers/watchman.js

+6-10
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,8 @@
99

1010
'use strict';
1111

12+
const H = require('../constants');
13+
1214
const denodeify = require('denodeify');
1315
const path = require('../fastpath');
1416
const watchman = require('fb-watchman');
@@ -29,7 +31,7 @@ function WatchmanError(error) {
2931
module.exports = function watchmanCrawl(
3032
roots,
3133
extensions,
32-
ignorePattern,
34+
ignore,
3335
data
3436
) {
3537
const files = data.files;
@@ -78,18 +80,12 @@ module.exports = function watchmanCrawl(
7880
response.files.forEach(fileData => {
7981
const name = root + path.sep + fileData.name;
8082
if (!fileData.exists) {
81-
//console.log('remove', name);
8283
delete files[name];
83-
} else if (!ignorePattern.test(name)) {
84-
//console.log('add', name);
84+
} else if (!ignore(name)) {
8585
const mtime = fileData.mtime_ms.toNumber();
86-
const isNew = !files[name] || files[name].mtime !== mtime;
86+
const isNew = !files[name] || files[name][H.MTIME] !== mtime;
8787
if (isNew) {
88-
files[name] = {
89-
id: null,
90-
mtime,
91-
visited: false,
92-
};
88+
files[name] = [0, mtime, 0, []];
9389
}
9490
}
9591
});

‎packages/jest-haste-map/src/index.js

+39-33
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,8 @@
88
*/
99
'use strict';
1010

11+
const H = require('./constants');
12+
1113
const crypto = require('crypto');
1214
const denodeify = require('denodeify');
1315
const execSync = require('child_process').execSync;
@@ -20,7 +22,6 @@ const watchmanCrawl = require('./crawlers/watchman');
2022
const worker = require('./worker');
2123
const workerFarm = require('worker-farm');
2224

23-
const GENERIC_PLATFORM = 'g';
2425
const NODE_MODULES = path.sep + 'node_modules' + path.sep;
2526
const VERSION = require('../package.json').version;
2627

@@ -42,6 +43,7 @@ class HasteMap {
4243
maxWorkers: options.maxWorkers,
4344
mocksPattern:
4445
options.mocksPattern ? new RegExp(options.mocksPattern) : null,
46+
name: options.name,
4547
platforms: options.platforms,
4648
resetCache: options.resetCache,
4749
roots: options.roots,
@@ -57,6 +59,7 @@ class HasteMap {
5759
this._cachePath = HasteMap.getCacheFilePath(
5860
this._options.cacheDirectory,
5961
VERSION,
62+
this._options.name,
6063
this._options.roots.join(':'),
6164
this._options.extensions.join(':'),
6265
this._options.platforms.join(':'),
@@ -121,17 +124,19 @@ class HasteMap {
121124
const mocksPattern = this._options.mocksPattern;
122125
const promises = [];
123126
const setModule = module => {
124-
if (!map[module.id]) {
125-
map[module.id] = Object.create(null);
127+
if (!map[module[H.ID]]) {
128+
map[module[H.ID]] = Object.create(null);
126129
}
127-
const moduleMap = map[module.id];
128-
const platform = getPlatformExtension(module.path) || GENERIC_PLATFORM;
130+
const moduleMap = map[module[H.ID]];
131+
const platform =
132+
getPlatformExtension(module[H.PATH]) || H.GENERIC_PLATFORM;
129133
const existingModule = moduleMap[platform];
130-
if (existingModule && existingModule.path !== module.path) {
134+
if (existingModule && existingModule[H.PATH] !== module[H.PATH]) {
131135
console.warn(
132136
`@providesModule naming collision:\n` +
133137
` Duplicate module name: ${module.id}\n` +
134-
` Paths: ${module.path} collides with ${existingModule.path}\n\n` +
138+
` Paths: ${module[H.PATH]} collides with ` +
139+
`${existingModule[H.PATH]}\n\n` +
135140
`This warning is caused by a @providesModule declaration ` +
136141
`with the same name accross two different files.`
137142
);
@@ -145,31 +150,27 @@ class HasteMap {
145150
mocks[path.basename(filePath, path.extname(filePath))] = filePath;
146151
}
147152

148-
if (!this._isNodeModulesDir(filePath)) {
149-
const fileData = data.files[filePath];
150-
const moduleData = data.map[fileData.id];
151-
if (fileData.visited) {
152-
if (!fileData.id) {
153-
continue;
154-
} else if (fileData.id && moduleData) {
155-
map[fileData.id] = moduleData;
156-
continue;
157-
}
153+
const fileData = data.files[filePath];
154+
const moduleData = data.map[fileData[H.ID]];
155+
if (fileData[H.VISITED]) {
156+
if (!fileData[H.ID]) {
157+
continue;
158+
} else if (fileData[H.ID] && moduleData) {
159+
map[fileData[H.ID]] = moduleData;
160+
continue;
158161
}
159-
160-
promises.push(
161-
this._getWorker()({filePath}).then(data => {
162-
fileData.visited = true;
163-
if (data.module) {
164-
fileData.id = data.module.id;
165-
setModule(data.module);
166-
}
167-
if (data.dependencies) {
168-
fileData.dependencies = data.dependencies;
169-
}
170-
})
171-
);
172162
}
163+
164+
promises.push(
165+
this._getWorker()({filePath}).then(data => {
166+
fileData[H.VISITED] = 1;
167+
if (data.module) {
168+
fileData[H.ID] = data.module[H.ID];
169+
setModule(data.module);
170+
}
171+
fileData[H.DEPENDENCIES] = data.dependencies || [];
172+
})
173+
);
173174
}
174175

175176
return Promise.all(promises)
@@ -228,11 +229,18 @@ class HasteMap {
228229
return crawl(
229230
this._options.roots,
230231
this._options.extensions,
231-
this._options.ignorePattern,
232+
this._ignore.bind(this),
232233
data
233234
);
234235
}
235236

237+
_ignore(filePath) {
238+
return (
239+
this._options.ignorePattern.test(filePath) ||
240+
this._isNodeModulesDir(filePath)
241+
);
242+
}
243+
236244
_isNodeModulesDir(filePath) {
237245
if (!filePath.includes(NODE_MODULES)) {
238246
return false;
@@ -257,6 +265,4 @@ class HasteMap {
257265

258266
}
259267

260-
HasteMap.GENERIC_PLATFORM = GENERIC_PLATFORM;
261-
262268
module.exports = HasteMap;

‎packages/jest-haste-map/src/worker.js

+4-10
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,8 @@
77
*/
88
'use strict';
99

10+
const H = require('./constants');
11+
1012
const docblock = require('./lib/docblock');
1113
const extractRequires = require('./lib/extractRequires');
1214
const fs = require('graceful-fs');
@@ -40,22 +42,14 @@ module.exports = (data, callback) => {
4042
if (filePath.endsWith(PACKAGE_JSON)) {
4143
const fileData = JSON.parse(content);
4244
if (fileData.name) {
43-
module = {
44-
id: fileData.name,
45-
path: filePath,
46-
type: 'package',
47-
};
45+
module = [fileData.name, filePath, H.PACKAGE];
4846
}
4947
} else {
5048
const doc = docblock.parse(docblock.extract(content));
5149
const id = doc.providesModule || doc.provides;
5250
dependencies = extractRequires(content);
5351
if (id) {
54-
module = {
55-
id,
56-
path: filePath,
57-
type: 'module',
58-
};
52+
module = [id, filePath, H.MODULE];
5953
}
6054
}
6155
callback(null, {module, dependencies});

‎src/Runtime/Runtime.js

+6-7
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,7 @@
88

99
'use strict';
1010

11-
const HasteMap = require('jest-haste-map');
11+
const H = require('jest-haste-map/src/constants');
1212

1313
const constants = require('../constants');
1414
const fs = require('graceful-fs');
@@ -456,23 +456,22 @@ class Runtime {
456456

457457
_getModule(name, type) {
458458
if (!type) {
459-
type = 'module';
459+
type = H.MODULE;
460460
}
461461

462462
const map = this._modules[name];
463463
if (map) {
464-
const module =
465-
map[this._defaultPlatform] || map[HasteMap.GENERIC_PLATFORM];
466-
if (module && module.type == type) {
467-
return module.path;
464+
const module = map[this._defaultPlatform] || map[H.GENERIC_PLATFORM];
465+
if (module && module[H.TYPE] == type) {
466+
return module[H.PATH];
468467
}
469468
}
470469

471470
return null;
472471
}
473472

474473
_getPackage(name) {
475-
return this._getModule(name, 'package');
474+
return this._getModule(name, H.PACKAGE);
476475
}
477476

478477
_getMockModule(name) {

‎src/TestRunner.js

+48-94
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
'use strict';
1010

11+
const H = require('jest-haste-map/src/constants');
1112
const Test = require('./Test');
1213

1314
const createHasteMap = require('./lib/createHasteMap');
@@ -16,6 +17,7 @@ const getCacheFilePath = require('jest-haste-map').getCacheFilePath;
1617
const mkdirp = require('mkdirp');
1718
const path = require('path');
1819
const promisify = require('./lib/promisify');
20+
const resolveNodeModule = require('./lib/resolveNodeModule');
1921
const utils = require('jest-util');
2022
const workerFarm = require('worker-farm');
2123

@@ -55,13 +57,18 @@ class TestRunner {
5557
}
5658
}
5759

58-
this._resolver = createHasteMap(config, {
60+
this._hasteMap = createHasteMap(config, {
5961
maxWorkers: options.runInBand ? 1 : this._opts.maxWorkers,
6062
resetCache: !config.cache,
6163
});
6264

6365
// warm-up and cache mocks
64-
this._resolver.build():
66+
console.time('build');
67+
this._hasteMap.build().then(data => {
68+
console.timeEnd('build');
69+
console.log('files', Object.keys(data.files).length);
70+
console.log('modules', Object.keys(data.map).length);
71+
});
6572

6673
this._testPathDirsRegExp = new RegExp(
6774
config.testPathDirs
@@ -84,7 +91,7 @@ class TestRunner {
8491
}
8592

8693
_getAllTestPaths() {
87-
return this._resolver
94+
return this._hasteMap
8895
.matchFiles(this._config.testDirectoryName)
8996
.then(paths => paths.filter(path => this.isTestFilePath(path)));
9097
}
@@ -107,15 +114,15 @@ class TestRunner {
107114
const visitedModules = new Set();
108115
while (changed.size) {
109116
changed = new Set(moduleMap.filter(module => (
110-
!visitedModules.has(module.path) &&
117+
!visitedModules.has(module.file) &&
111118
module.dependencies.some(dep => dep && changed.has(dep))
112119
)).map(module => {
113-
const path = module.path;
114-
if (this.isTestFilePath(path)) {
115-
relatedPaths.add(path);
120+
const file = module.file;
121+
if (this.isTestFilePath(file)) {
122+
relatedPaths.add(file);
116123
}
117-
visitedModules.add(path);
118-
return module.name;
124+
visitedModules.add(file);
125+
return module.file;
119126
}));
120127
}
121128
return relatedPaths;
@@ -125,102 +132,49 @@ class TestRunner {
125132
if (!changedPaths.size) {
126133
return Promise.resolve([]);
127134
}
135+
128136
const relatedPaths = new Set();
129-
return this._resolver.getAllModules().then(allModules => {
137+
return this._hasteMap.build().then(data => {
130138
const changed = new Set();
131139
for (const path of changedPaths) {
132140
if (fileExists(path)) {
133-
const module = this._resolver.getModuleForPath(path);
141+
const module = data.files[path];
134142
if (module) {
135-
changed.add(module.path);
136-
if (this.isTestFilePath(module.path)) {
137-
relatedPaths.add(module.path);
143+
changed.add(path);
144+
if (this.isTestFilePath(path)) {
145+
relatedPaths.add(path);
138146
}
139147
}
140148
}
141149
}
142-
return Promise.all(Object.keys(allModules).map(path =>
143-
this._resolver.getShallowDependencies(path)
144-
.then(response => ({
145-
name: path,
146-
path,
147-
dependencies: response.dependencies.map(dep => dep.path),
148-
}))
149-
)).then(moduleMap => Array.from(this.collectChangedModules(
150-
relatedPaths,
151-
moduleMap,
152-
changed
153-
)));
154-
});
155-
}
156-
157-
promiseHasteTestPathsRelatedTo(changedPaths) {
158-
if (!changedPaths.size) {
159-
return Promise.resolve([]);
160-
}
161150

162-
return Promise.all([
163-
this._getAllTestPaths(),
164-
this._resolver.build(),
165-
]).then(response => {
166-
const testPaths = response[0];
167-
const hasteMap = response[1];
168-
const relatedPaths = new Set();
169-
const changed = new Set();
151+
const platform = this._config.haste.defaultPlatform;
152+
const extensions =
153+
this._config.moduleFileExtensions.map(ext => '.' + ext);
170154
const moduleMap = [];
171-
testPaths.forEach(path => {
172-
if (changedPaths.has(path) && this.isTestFilePath(path)) {
173-
relatedPaths.add(path);
174-
}
175-
moduleMap.push({name: path, path, dependencies: null});
176-
});
177-
const collectModules = list => {
178-
for (const name in list) {
179-
const path = list[name];
180-
if (changedPaths.has(path)) {
181-
changed.add(name);
182-
if (this.isTestFilePath(path)) {
183-
relatedPaths.add(path);
184-
}
155+
for (const file in data.files) {
156+
const fileData = data.files[file];
157+
const dependencies = fileData[H.DEPENDENCIES].map(dep => {
158+
const map = data.map[dep];
159+
if (data.map[dep]) {
160+
const module =
161+
map[platform] || map[H.GENERIC_PLATFORM];
162+
return module && module[H.PATH];
163+
} else {
164+
return resolveNodeModule(dep, path.dirname(file), extensions);
185165
}
186-
moduleMap.push({name, path, dependencies: null});
187-
}
188-
};
189-
collectModules(hasteMap.modules);
190-
collectModules(hasteMap.mocks);
191-
192-
const deferreds = moduleMap.map(() => {
193-
let resolve;
194-
const promise = new Promise(_resolve => resolve = _resolve);
195-
return {resolve, promise};
196-
});
197-
let i = 0;
198-
const nextResolution = () => {
199-
if (i >= moduleMap.length) {
200-
return;
201-
}
202-
203-
const currentIndex = i;
204-
const module = moduleMap[currentIndex];
205-
const deferred = deferreds[currentIndex];
206-
i++;
207-
this._resolver.getModuleForPath(module.path).getDependencies()
208-
.then(dependencies => {
209-
nextResolution();
210-
moduleMap[currentIndex].dependencies = dependencies;
211-
})
212-
.then(() => deferred.resolve());
213-
};
214-
215-
for (let i = 0; i < 20; i++) {
216-
nextResolution();
166+
}).filter(
167+
dep => !!dep
168+
);
169+
moduleMap.push({file, dependencies});
170+
// should take moduleNameMapper into account
217171
}
218-
return Promise.all(deferreds.map(deferred => deferred.promise))
219-
.then(() => Array.from(this.collectChangedModules(
220-
relatedPaths,
221-
moduleMap,
222-
changed
223-
)));
172+
173+
return Array.from(this.collectChangedModules(
174+
relatedPaths,
175+
moduleMap,
176+
changed
177+
));
224178
});
225179
}
226180

@@ -386,7 +340,7 @@ class TestRunner {
386340
_createInBandTestRun(testPaths, onTestResult, onRunFailure) {
387341
return testPaths.reduce((promise, path) =>
388342
promise
389-
.then(() => this._resolver.build())
343+
.then(() => this._hasteMap.build())
390344
.then(moduleMap => new Test(path, this._config, moduleMap).run())
391345
.then(result => onTestResult(path, result))
392346
.catch(err => onRunFailure(path, err)),
@@ -396,7 +350,7 @@ class TestRunner {
396350

397351
_createParallelTestRun(testPaths, onTestResult, onRunFailure) {
398352
const config = this._config;
399-
return this._resolver.build()
353+
return this._hasteMap.build()
400354
.then(() => {
401355
const farm = workerFarm({
402356
autoStart: true,

‎src/__tests__/TestRunner-test.js

+6-10
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,6 @@ describe('TestRunner', () => {
2626
describe('isTestFilePath', () => {
2727

2828
beforeEach(() => {
29-
jest.mock('../resolvers/HasteResolver');
3029
runner = new TestRunner(normalizeConfig({
3130
cacheDirectory: global.CACHE_DIRECTORY,
3231
name,
@@ -78,18 +77,16 @@ describe('TestRunner', () => {
7877
});
7978

8079
beforeEach(() => {
81-
jest.dontMock('../resolvers/HasteResolver');
82-
runner = new TestRunner(config, {});
80+
runner = new TestRunner(config, {
81+
maxWorkers: 1,
82+
});
8383
});
8484

8585
pit('makes sure a file is related to itself', () => {
86-
const path = rootPath;
87-
88-
return runner.promiseTestPathsRelatedTo(new Set([path]))
86+
return runner.promiseTestPathsRelatedTo(new Set([rootPath]))
8987
.then(relatedTests => {
9088
expect(relatedTests).toEqual([rootPath]);
91-
})
92-
.then(() => runner.end());
89+
});
9390
});
9491

9592
pit('finds tests that depend directly on the path', () => {
@@ -102,8 +99,7 @@ describe('TestRunner', () => {
10299
filePath,
103100
rootPath,
104101
]);
105-
})
106-
.then(() => runner.end());
102+
});
107103
});
108104
});
109105
});

‎src/lib/createHasteMap.js

+1
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ module.exports = function createHasteMap(config, options) {
2424
ignorePattern,
2525
maxWorkers: options && options.maxWorkers,
2626
mocksPattern: config.mocksPattern,
27+
name: config.name,
2728
platforms: config.haste.platforms || ['ios', 'android'],
2829
providesModuleNodeModules: config.haste.providesModuleNodeModules,
2930
resetCache: options && options.resetCache,

0 commit comments

Comments
 (0)
Please sign in to comment.