From 52018aa253817a7c473ca35f24f9707f295d1182 Mon Sep 17 00:00:00 2001 From: Chunpeng Huo Date: Sat, 18 Jan 2020 11:44:58 +1100 Subject: [PATCH] feat: support option noJsx and noTypeScript to skip those parsers This is to give modify-code a chance to bypass a babel performance issue babel/babel#11029. --- index.js | 4 +- test/modify-code.spec.js | 52 +++++++++++++++++++++++- test/tokenize.spec.js | 88 +++++++++++++++++++++++++++++++++++++++- tokenize.js | 64 ++++++++++++++++------------- 4 files changed, 175 insertions(+), 33 deletions(-) diff --git a/index.js b/index.js index 01bffda..4b7b342 100644 --- a/index.js +++ b/index.js @@ -2,7 +2,7 @@ var tokenize = require('./tokenize'); var SourceNode = require('source-map').SourceNode; exports.__esModule = true; -exports['default'] = function(code, filePath) { +exports['default'] = function(code, filePath, options) { // the file name to be used in sourcemap sources and file fields filePath = (filePath || 'file.js').replace(/\\/g, '/'); var mutations = []; @@ -82,7 +82,7 @@ exports['default'] = function(code, filePath) { var ms = compactMutations(mutations); var i = 0, ti = 0, ii = ms.length, newTokens = []; var m, offset, offset2, merged, isInsertion; - var tokens = tokenize(code); + var tokens = tokenize(code, options); for (; i < ii; i++) { m = ms[i]; diff --git a/test/modify-code.spec.js b/test/modify-code.spec.js index 5159bde..82b628a 100644 --- a/test/modify-code.spec.js +++ b/test/modify-code.spec.js @@ -577,7 +577,7 @@ test('modify-code can chain mutation calls, in different order', function(t) { }); test('modify-code prepends, replaces and inserts at adjacent positions', function(t) { - var m = modify("import { a } from 'foo';@a() export class B {}", 'some.js'); + var m = modify("import { a } from 'foo';@a() export class B {}", 'some.js', {noJsx: true, noTypeScript: true}); // prepend import m.prepend("import vf from './some.html';\n") // rewrite import @@ -641,3 +641,53 @@ test('modify-code prepends and inserts around spaces', function(t) { }); t.end(); }); + +test('modify-code mutates jsx and typescript code', function(t) { + var m = modify('export default (name: string) =>

{name}

;', 'some.js'); + m.prepend("import React from 'react';\n"); + + t.deepEqual(m.transform(), { + code: "import React from 'react';\nexport default (name: string) =>

{name}

;", + map: { + version: 3, + sources: ['some.js'], + sourcesContent: ['export default (name: string) =>

{name}

;'], + file: 'some.js', + names: [], + mappings: encode([ [ [ 0, 0, 0, 0 ] ], + [ + [ 0, 0, 0, 0 ], [ 6, 0, 0, 6 ], + [ 7, 0, 0, 7 ], [ 14, 0, 0, 14 ], + [ 15, 0, 0, 15 ], [ 16, 0, 0, 16 ], + [ 20, 0, 0, 20 ], [ 21, 0, 0, 21 ], + [ 22, 0, 0, 22 ], [ 28, 0, 0, 28 ], + [ 29, 0, 0, 29 ], [ 30, 0, 0, 30 ], + [ 32, 0, 0, 32 ], [ 33, 0, 0, 33 ], + [ 34, 0, 0, 34 ], [ 35, 0, 0, 35 ], + [ 36, 0, 0, 36 ], [ 37, 0, 0, 37 ], + [ 41, 0, 0, 41 ], [ 42, 0, 0, 42 ], + [ 43, 0, 0, 43 ], [ 44, 0, 0, 44 ], + [ 45, 0, 0, 45 ], [ 46, 0, 0, 46 ] + ] + ]) + } + }); + t.end(); +}); + +test('modify-code can turn off jsx', function(t) { + var m = modify('export default (name: string) =>

{name}

;', 'some.js', {noJsx: true}); + m.prepend("import React from 'react';\n"); + + t.throws(function() {m.transform();}); + t.end(); +}); + +test('modify-code can turn off typescript', function(t) { + var m = modify('export default (name: string) =>

{name}

;', 'some.js', {noTypeScript: true}); + m.prepend("import React from 'react';\n"); + + t.throws(function() {m.transform();}); + t.end(); +}); + diff --git a/test/tokenize.spec.js b/test/tokenize.spec.js index 7b110ae..ca8080b 100644 --- a/test/tokenize.spec.js +++ b/test/tokenize.spec.js @@ -77,7 +77,7 @@ test('tokenize outputs tokens with leading and tailing white spaces', function(t t.end(); }); -test('tokenize understand latest syntax', function(t) { +test('tokenize understands latest syntax', function(t) { var code = '@a()\nexport class B {\n c = 1;\n}\n'; var tokens = tokenize(code); t.deepEqual(tokens, [ @@ -108,7 +108,7 @@ test('tokenize understand latest syntax', function(t) { t.end(); }); -test('tokenize understand jsx and typescript syntax', function(t) { +test('tokenize understands jsx syntax', function(t) { var code = 'export default () => ;'; var tokens = tokenize(code); t.deepEqual(tokens, [ @@ -143,3 +143,87 @@ test('tokenize understand jsx and typescript syntax', function(t) { t.equal(tokensToCode(tokens), code); t.end(); }); + +test('tokenize can turn off jsx syntax', function(t) { + var code = 'export default () => ;'; + t.throws(function() { tokenize(code, {noJsx: true}); }); + t.end(); +}); + +test('tokenize still understands jsx syntax when typescript is turned off', function(t) { + var code = 'export default () => ;'; + var tokens = tokenize(code, {noTypeScript: true}); + t.deepEqual(tokens, [ + { value: 'export', start: 0, end: 6, line: 1, column: 0 }, + { value: ' ', start: 6, end: 7, line: 1, column: 6 }, + { value: 'default', start: 7, end: 14, line: 1, column: 7 }, + { value: ' ', start: 14, end: 15, line: 1, column: 14 }, + { value: '(', start: 15, end: 16, line: 1, column: 15 }, + { value: ')', start: 16, end: 17, line: 1, column: 16 }, + { value: ' ', start: 17, end: 18, line: 1, column: 17 }, + { value: '=>', start: 18, end: 20, line: 1, column: 18 }, + { value: ' ', start: 20, end: 21, line: 1, column: 20 }, + { value: '<', start: 21, end: 22, line: 1, column: 21 }, + { value: 'button', start: 22, end: 28, line: 1, column: 22 }, + { value: ' ', start: 28, end: 29, line: 1, column: 28 }, + { value: 'onClick', start: 29, end: 36, line: 1, column: 29 }, + { value: '=', start: 36, end: 37, line: 1, column: 36 }, + { value: '{', start: 37, end: 38, line: 1, column: 37 }, + { value: 'props', start: 38, end: 43, line: 1, column: 38 }, + { value: '.', start: 43, end: 44, line: 1, column: 43 }, + { value: 'onClick', start: 44, end: 51, line: 1, column: 44 }, + { value: '}', start: 51, end: 52, line: 1, column: 51 }, + { value: '>', start: 52, end: 53, line: 1, column: 52 }, + { value: 'OK', start: 53, end: 55, line: 1, column: 53 }, + { value: '<', start: 55, end: 56, line: 1, column: 55 }, + { value: '/', start: 56, end: 57, line: 1, column: 56 }, + { value: 'button', start: 57, end: 63, line: 1, column: 57 }, + { value: '>', start: 63, end: 64, line: 1, column: 63 }, + { value: ';', start: 64, end: 65, line: 1, column: 64 } + ] +); + t.equal(tokensToCode(tokens), code); + t.end(); +}); + + +test('tokenize understand jsx and typescript syntax', function(t) { + var code = 'export default (name: string) =>

{name}

;'; + var tokens = tokenize(code); + t.deepEqual(tokens, [ + { value: 'export', start: 0, end: 6, line: 1, column: 0 }, + { value: ' ', start: 6, end: 7, line: 1, column: 6 }, + { value: 'default', start: 7, end: 14, line: 1, column: 7 }, + { value: ' ', start: 14, end: 15, line: 1, column: 14 }, + { value: '(', start: 15, end: 16, line: 1, column: 15 }, + { value: 'name', start: 16, end: 20, line: 1, column: 16 }, + { value: ':', start: 20, end: 21, line: 1, column: 20 }, + { value: ' ', start: 21, end: 22, line: 1, column: 21 }, + { value: 'string', start: 22, end: 28, line: 1, column: 22 }, + { value: ')', start: 28, end: 29, line: 1, column: 28 }, + { value: ' ', start: 29, end: 30, line: 1, column: 29 }, + { value: '=>', start: 30, end: 32, line: 1, column: 30 }, + { value: ' ', start: 32, end: 33, line: 1, column: 32 }, + { value: '<', start: 33, end: 34, line: 1, column: 33 }, + { value: 'p', start: 34, end: 35, line: 1, column: 34 }, + { value: '>', start: 35, end: 36, line: 1, column: 35 }, + { value: '{', start: 36, end: 37, line: 1, column: 36 }, + { value: 'name', start: 37, end: 41, line: 1, column: 37 }, + { value: '}', start: 41, end: 42, line: 1, column: 41 }, + { value: '<', start: 42, end: 43, line: 1, column: 42 }, + { value: '/', start: 43, end: 44, line: 1, column: 43 }, + { value: 'p', start: 44, end: 45, line: 1, column: 44 }, + { value: '>', start: 45, end: 46, line: 1, column: 45 }, + { value: ';', start: 46, end: 47, line: 1, column: 46 } + ] +); + t.equal(tokensToCode(tokens), code); + t.end(); +}); + +test('tokenize can turn off typescript syntax', function(t) { + var code = 'export default (name: string) =>

{name}

;'; + t.throws(function() { tokenize(code, {noTypeScript: true}); }); + t.end(); +}); + diff --git a/tokenize.js b/tokenize.js index f8bcd77..0db0b8c 100644 --- a/tokenize.js +++ b/tokenize.js @@ -1,33 +1,41 @@ var parser = require('@babel/parser'); -module.exports = function(code) { - var tokens = parser.parse(code, {sourceType: 'module', plugins: [ - 'jsx', - 'typescript', - 'asyncGenerators', - 'bigInt', - 'classProperties', - 'classPrivateProperties', - 'classPrivateMethods', - 'decorators-legacy', - // ['decorators', {'decoratorsBeforeExport': true}], - 'doExpressions', - 'dynamicImport', - 'exportDefaultFrom', - 'exportNamespaceFrom', - 'functionBind', - 'functionSent', - 'importMeta', - 'logicalAssignment', - 'nullishCoalescingOperator', - 'numericSeparator', - 'objectRestSpread', - 'optionalCatchBinding', - 'optionalChaining', - 'partialApplication', - // ['pipelineOperator', {proposal: 'minimal'}], - 'throwExpressions', - ], tokens: true}).tokens; +var commonPlugins = [ + 'asyncGenerators', + 'bigInt', + 'classProperties', + 'classPrivateProperties', + 'classPrivateMethods', + 'decorators-legacy', + // ['decorators', {'decoratorsBeforeExport': true}], + 'doExpressions', + 'dynamicImport', + 'exportDefaultFrom', + 'exportNamespaceFrom', + 'functionBind', + 'functionSent', + 'importMeta', + 'logicalAssignment', + 'nullishCoalescingOperator', + 'numericSeparator', + 'objectRestSpread', + 'optionalCatchBinding', + 'optionalChaining', + 'partialApplication', + // ['pipelineOperator', {proposal: 'minimal'}], + 'throwExpressions', +]; + +module.exports = function(code, options) { + var plugins = commonPlugins.slice(0); + if (!(options && options.noJsx)) plugins.push('jsx'); + if (!(options && options.noTypeScript)) plugins.push('typescript'); + + var tokens = parser.parse(code, { + sourceType: 'module', + plugins: plugins, + tokens: true + }).tokens; var i = 0, ii = tokens.length, fullTokens = [], token, lastToken;