Skip to content

Commit

Permalink
feat: support option noJsx and noTypeScript to skip those parsers
Browse files Browse the repository at this point in the history 
This is to give modify-code a chance to bypass a babel performance issue babel/babel#11029.
  • Loading branch information
@3cp
3cp committed Jan 18, 2020
1 parent 8223674 commit 52018aa
Show file tree
Hide file tree
Showing 4 changed files with 175 additions and 33 deletions.
4 changes: 2 additions & 2 deletions index.js
View file
Expand Up @@ -2,7 +2,7 @@ var tokenize = require('./tokenize');
var SourceNode = require('source-map').SourceNode;

exports.__esModule = true;
exports['default'] = function(code, filePath) {
exports['default'] = function(code, filePath, options) {
// the file name to be used in sourcemap sources and file fields
filePath = (filePath || 'file.js').replace(/\\/g, '/');
var mutations = [];
Expand Down Expand Up @@ -82,7 +82,7 @@ exports['default'] = function(code, filePath) {
var ms = compactMutations(mutations);
var i = 0, ti = 0, ii = ms.length, newTokens = [];
var m, offset, offset2, merged, isInsertion;
var tokens = tokenize(code);
var tokens = tokenize(code, options);

for (; i < ii; i++) {
m = ms[i];
Expand Down
52 changes: 51 additions & 1 deletion test/modify-code.spec.js
View file
Expand Up @@ -577,7 +577,7 @@ test('modify-code can chain mutation calls, in different order', function(t) {
});

test('modify-code prepends, replaces and inserts at adjacent positions', function(t) {
var m = modify("import { a } from 'foo';@a() export class B {}", 'some.js');
var m = modify("import { a } from 'foo';@a() export class B {}", 'some.js', {noJsx: true, noTypeScript: true});
// prepend import
m.prepend("import vf from './some.html';\n")
// rewrite import
Expand Down Expand Up @@ -641,3 +641,53 @@ test('modify-code prepends and inserts around spaces', function(t) {
});
t.end();
});

test('modify-code mutates jsx and typescript code', function(t) {
var m = modify('export default (name: string) => <p>{name}</p>;', 'some.js');
m.prepend("import React from 'react';\n");

t.deepEqual(m.transform(), {
code: "import React from 'react';\nexport default (name: string) => <p>{name}</p>;",
map: {
version: 3,
sources: ['some.js'],
sourcesContent: ['export default (name: string) => <p>{name}</p>;'],
file: 'some.js',
names: [],
mappings: encode([ [ [ 0, 0, 0, 0 ] ],
[
[ 0, 0, 0, 0 ], [ 6, 0, 0, 6 ],
[ 7, 0, 0, 7 ], [ 14, 0, 0, 14 ],
[ 15, 0, 0, 15 ], [ 16, 0, 0, 16 ],
[ 20, 0, 0, 20 ], [ 21, 0, 0, 21 ],
[ 22, 0, 0, 22 ], [ 28, 0, 0, 28 ],
[ 29, 0, 0, 29 ], [ 30, 0, 0, 30 ],
[ 32, 0, 0, 32 ], [ 33, 0, 0, 33 ],
[ 34, 0, 0, 34 ], [ 35, 0, 0, 35 ],
[ 36, 0, 0, 36 ], [ 37, 0, 0, 37 ],
[ 41, 0, 0, 41 ], [ 42, 0, 0, 42 ],
[ 43, 0, 0, 43 ], [ 44, 0, 0, 44 ],
[ 45, 0, 0, 45 ], [ 46, 0, 0, 46 ]
]
])
}
});
t.end();
});

test('modify-code can turn off jsx', function(t) {
var m = modify('export default (name: string) => <p>{name}</p>;', 'some.js', {noJsx: true});
m.prepend("import React from 'react';\n");

t.throws(function() {m.transform();});
t.end();
});

test('modify-code can turn off typescript', function(t) {
var m = modify('export default (name: string) => <p>{name}</p>;', 'some.js', {noTypeScript: true});
m.prepend("import React from 'react';\n");

t.throws(function() {m.transform();});
t.end();
});

88 changes: 86 additions & 2 deletions test/tokenize.spec.js
View file
Expand Up @@ -77,7 +77,7 @@ test('tokenize outputs tokens with leading and tailing white spaces', function(t
t.end();
});

test('tokenize understand latest syntax', function(t) {
test('tokenize understands latest syntax', function(t) {
var code = '@a()\nexport class B {\n c = 1;\n}\n';
var tokens = tokenize(code);
t.deepEqual(tokens, [
Expand Down Expand Up @@ -108,7 +108,7 @@ test('tokenize understand latest syntax', function(t) {
t.end();
});

test('tokenize understand jsx and typescript syntax', function(t) {
test('tokenize understands jsx syntax', function(t) {
var code = 'export default () => <button onClick={props.onClick}>OK</button>;';
var tokens = tokenize(code);
t.deepEqual(tokens, [
Expand Down Expand Up @@ -143,3 +143,87 @@ test('tokenize understand jsx and typescript syntax', function(t) {
t.equal(tokensToCode(tokens), code);
t.end();
});

test('tokenize can turn off jsx syntax', function(t) {
var code = 'export default () => <button onClick={props.onClick}>OK</button>;';
t.throws(function() { tokenize(code, {noJsx: true}); });
t.end();
});

test('tokenize still understands jsx syntax when typescript is turned off', function(t) {
var code = 'export default () => <button onClick={props.onClick}>OK</button>;';
var tokens = tokenize(code, {noTypeScript: true});
t.deepEqual(tokens, [
{ value: 'export', start: 0, end: 6, line: 1, column: 0 },
{ value: ' ', start: 6, end: 7, line: 1, column: 6 },
{ value: 'default', start: 7, end: 14, line: 1, column: 7 },
{ value: ' ', start: 14, end: 15, line: 1, column: 14 },
{ value: '(', start: 15, end: 16, line: 1, column: 15 },
{ value: ')', start: 16, end: 17, line: 1, column: 16 },
{ value: ' ', start: 17, end: 18, line: 1, column: 17 },
{ value: '=>', start: 18, end: 20, line: 1, column: 18 },
{ value: ' ', start: 20, end: 21, line: 1, column: 20 },
{ value: '<', start: 21, end: 22, line: 1, column: 21 },
{ value: 'button', start: 22, end: 28, line: 1, column: 22 },
{ value: ' ', start: 28, end: 29, line: 1, column: 28 },
{ value: 'onClick', start: 29, end: 36, line: 1, column: 29 },
{ value: '=', start: 36, end: 37, line: 1, column: 36 },
{ value: '{', start: 37, end: 38, line: 1, column: 37 },
{ value: 'props', start: 38, end: 43, line: 1, column: 38 },
{ value: '.', start: 43, end: 44, line: 1, column: 43 },
{ value: 'onClick', start: 44, end: 51, line: 1, column: 44 },
{ value: '}', start: 51, end: 52, line: 1, column: 51 },
{ value: '>', start: 52, end: 53, line: 1, column: 52 },
{ value: 'OK', start: 53, end: 55, line: 1, column: 53 },
{ value: '<', start: 55, end: 56, line: 1, column: 55 },
{ value: '/', start: 56, end: 57, line: 1, column: 56 },
{ value: 'button', start: 57, end: 63, line: 1, column: 57 },
{ value: '>', start: 63, end: 64, line: 1, column: 63 },
{ value: ';', start: 64, end: 65, line: 1, column: 64 }
]
);
t.equal(tokensToCode(tokens), code);
t.end();
});


test('tokenize understand jsx and typescript syntax', function(t) {
var code = 'export default (name: string) => <p>{name}</p>;';
var tokens = tokenize(code);
t.deepEqual(tokens, [
{ value: 'export', start: 0, end: 6, line: 1, column: 0 },
{ value: ' ', start: 6, end: 7, line: 1, column: 6 },
{ value: 'default', start: 7, end: 14, line: 1, column: 7 },
{ value: ' ', start: 14, end: 15, line: 1, column: 14 },
{ value: '(', start: 15, end: 16, line: 1, column: 15 },
{ value: 'name', start: 16, end: 20, line: 1, column: 16 },
{ value: ':', start: 20, end: 21, line: 1, column: 20 },
{ value: ' ', start: 21, end: 22, line: 1, column: 21 },
{ value: 'string', start: 22, end: 28, line: 1, column: 22 },
{ value: ')', start: 28, end: 29, line: 1, column: 28 },
{ value: ' ', start: 29, end: 30, line: 1, column: 29 },
{ value: '=>', start: 30, end: 32, line: 1, column: 30 },
{ value: ' ', start: 32, end: 33, line: 1, column: 32 },
{ value: '<', start: 33, end: 34, line: 1, column: 33 },
{ value: 'p', start: 34, end: 35, line: 1, column: 34 },
{ value: '>', start: 35, end: 36, line: 1, column: 35 },
{ value: '{', start: 36, end: 37, line: 1, column: 36 },
{ value: 'name', start: 37, end: 41, line: 1, column: 37 },
{ value: '}', start: 41, end: 42, line: 1, column: 41 },
{ value: '<', start: 42, end: 43, line: 1, column: 42 },
{ value: '/', start: 43, end: 44, line: 1, column: 43 },
{ value: 'p', start: 44, end: 45, line: 1, column: 44 },
{ value: '>', start: 45, end: 46, line: 1, column: 45 },
{ value: ';', start: 46, end: 47, line: 1, column: 46 }
]
);
t.equal(tokensToCode(tokens), code);
t.end();
});

test('tokenize can turn off typescript syntax', function(t) {
var code = 'export default (name: string) => <p>{name}</p>;';
t.throws(function() { tokenize(code, {noTypeScript: true}); });
t.end();
});

64 changes: 36 additions & 28 deletions tokenize.js
View file
@@ -1,33 +1,41 @@
var parser = require('@babel/parser');

module.exports = function(code) {
var tokens = parser.parse(code, {sourceType: 'module', plugins: [
'jsx',
'typescript',
'asyncGenerators',
'bigInt',
'classProperties',
'classPrivateProperties',
'classPrivateMethods',
'decorators-legacy',
// ['decorators', {'decoratorsBeforeExport': true}],
'doExpressions',
'dynamicImport',
'exportDefaultFrom',
'exportNamespaceFrom',
'functionBind',
'functionSent',
'importMeta',
'logicalAssignment',
'nullishCoalescingOperator',
'numericSeparator',
'objectRestSpread',
'optionalCatchBinding',
'optionalChaining',
'partialApplication',
// ['pipelineOperator', {proposal: 'minimal'}],
'throwExpressions',
], tokens: true}).tokens;
var commonPlugins = [
'asyncGenerators',
'bigInt',
'classProperties',
'classPrivateProperties',
'classPrivateMethods',
'decorators-legacy',
// ['decorators', {'decoratorsBeforeExport': true}],
'doExpressions',
'dynamicImport',
'exportDefaultFrom',
'exportNamespaceFrom',
'functionBind',
'functionSent',
'importMeta',
'logicalAssignment',
'nullishCoalescingOperator',
'numericSeparator',
'objectRestSpread',
'optionalCatchBinding',
'optionalChaining',
'partialApplication',
// ['pipelineOperator', {proposal: 'minimal'}],
'throwExpressions',
];

module.exports = function(code, options) {
var plugins = commonPlugins.slice(0);
if (!(options && options.noJsx)) plugins.push('jsx');
if (!(options && options.noTypeScript)) plugins.push('typescript');

var tokens = parser.parse(code, {
sourceType: 'module',
plugins: plugins,
tokens: true
}).tokens;

var i = 0, ii = tokens.length, fullTokens = [], token, lastToken;

Expand Down

0 comments on commit 52018aa

Please sign in to comment.