From 3d5b3fe1633d1eb834c5c3e5e4ab94a1845f84ff Mon Sep 17 00:00:00 2001 From: Chunpeng Huo Date: Sat, 18 Jan 2020 00:16:44 +1100 Subject: [PATCH 1/5] fix(babel-parser): avoid state.clone() to clone the whole token store Fixed the performance issue on large input when turned on option {tokens: true} and typescript plugin which uses quite a few state.clone(). --- packages/babel-parser/src/parser/statement.js | 2 +- packages/babel-parser/src/tokenizer/index.js | 8 ++++++-- packages/babel-parser/src/tokenizer/state.js | 4 ---- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/babel-parser/src/parser/statement.js b/packages/babel-parser/src/parser/statement.js index a57a618c9748..76b0eb5ed402 100644 --- a/packages/babel-parser/src/parser/statement.js +++ b/packages/babel-parser/src/parser/statement.js @@ -67,7 +67,7 @@ export default class StatementParser extends ExpressionParser { file.program = this.finishNode(program, "Program"); file.comments = this.state.comments; - if (this.options.tokens) file.tokens = this.state.tokens; + if (this.options.tokens) file.tokens = this.tokens; return this.finishNode(file, "File"); } diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index a4e2de67a242..b3cd773e3bcb 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -1,6 +1,7 @@ // @flow import type { Options } from "../options"; +import * as N from "../types"; import type { Position } from "../util/location"; import * as charCodes from "charcodes"; import { isIdentifierStart, isIdentifierChar } from "../util/identifier"; @@ -114,6 +115,9 @@ export default class Tokenizer extends LocationParser { isLookahead: boolean; + // Token store. + tokens: Array = []; + constructor(options: Options, input: string) { super(); this.state = new State(); @@ -129,7 +133,7 @@ export default class Tokenizer extends LocationParser { if (!this.isLookahead) { this.checkKeywordEscapes(); if (this.options.tokens) { - this.state.tokens.push(new Token(this.state)); + this.tokens.push(new Token(this.state)); } } @@ -242,7 +246,7 @@ export default class Tokenizer extends LocationParser { loc: new SourceLocation(startLoc, endLoc), }; - if (this.options.tokens) this.state.tokens.push(comment); + if (this.options.tokens) this.tokens.push(comment); this.state.comments.push(comment); this.addComment(comment); } diff --git a/packages/babel-parser/src/tokenizer/state.js b/packages/babel-parser/src/tokenizer/state.js index ff0e704b8f00..da9bb6114dff 100644 --- a/packages/babel-parser/src/tokenizer/state.js +++ b/packages/babel-parser/src/tokenizer/state.js @@ -5,7 +5,6 @@ import * as N from "../types"; import { Position } from "../util/location"; import { types as ct, type TokContext } from "./context"; -import type { Token } from "./index"; import { types as tt, type TokenType } from "./types"; type TopicContextState = { @@ -93,9 +92,6 @@ export default class State { yieldPos: number = -1; awaitPos: number = -1; - // Token store. - tokens: Array = []; - // Comment store. comments: Array = []; From d7bf6e30ef4a91ff52850093b5f35ffe9bc0fee6 Mon Sep 17 00:00:00 2001 From: Chunpeng Huo Date: Sat, 18 Jan 2020 14:23:06 +1100 Subject: [PATCH 2/5] test(babel-parser): turn on 2 typescript tests with tokens:true The output.json is generated by old master to make sure no regression. --- .../input.ts | 0 .../async-generic-tokens-true/options.json | 5 + .../async-generic-tokens-true/output.json | 623 ++++++++++++++ .../arrow-function/async-generic/output.json | 231 ----- .../input.ts | 0 .../generic-complex-tokens-true/options.json | 5 + .../generic-complex-tokens-true/output.json | 804 ++++++++++++++++++ .../type-alias/generic-complex/output.json | 276 ------ 8 files changed, 1437 insertions(+), 507 deletions(-) rename packages/babel-parser/test/fixtures/typescript/arrow-function/{async-generic => async-generic-tokens-true}/input.ts (100%) create mode 100644 packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/options.json create mode 100644 packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json delete mode 100644 packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic/output.json rename packages/babel-parser/test/fixtures/typescript/type-alias/{generic-complex => generic-complex-tokens-true}/input.ts (100%) create mode 100644 packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/options.json create mode 100644 packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json delete mode 100644 packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex/output.json diff --git a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic/input.ts b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/input.ts similarity index 100% rename from packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic/input.ts rename to packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/input.ts diff --git a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/options.json b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/options.json new file mode 100644 index 000000000000..359dbf5e9514 --- /dev/null +++ b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/options.json @@ -0,0 +1,5 @@ +{ + "sourceType": "module", + "plugins": ["typescript"], + "tokens": true +} diff --git a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json new file mode 100644 index 000000000000..5a23156c9b3b --- /dev/null +++ b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json @@ -0,0 +1,623 @@ +{ + "type": "File", + "start": 0, + "end": 24, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 24 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 24, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 24 + } + }, + "sourceType": "module", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start": 0, + "end": 24, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 24 + } + }, + "expression": { + "type": "ArrowFunctionExpression", + "start": 0, + "end": 23, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 23 + } + }, + "typeParameters": { + "type": "TSTypeParameterDeclaration", + "start": 6, + "end": 9, + "loc": { + "start": { + "line": 1, + "column": 6 + }, + "end": { + "line": 1, + "column": 9 + } + }, + "params": [ + { + "type": "TSTypeParameter", + "start": 7, + "end": 8, + "loc": { + "start": { + "line": 1, + "column": 7 + }, + "end": { + "line": 1, + "column": 8 + } + }, + "name": "T" + } + ] + }, + "params": [ + { + "type": "Identifier", + "start": 10, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 10 + }, + "end": { + "line": 1, + "column": 14 + }, + "identifierName": "a" + }, + "name": "a", + "typeAnnotation": { + "type": "TSTypeAnnotation", + "start": 11, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 11 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "typeAnnotation": { + "type": "TSTypeReference", + "start": 13, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 13 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "typeName": { + "type": "Identifier", + "start": 13, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 13 + }, + "end": { + "line": 1, + "column": 14 + }, + "identifierName": "T" + }, + "name": "T" + } + } + } + } + ], + "returnType": { + "type": "TSTypeAnnotation", + "start": 15, + "end": 18, + "loc": { + "start": { + "line": 1, + "column": 15 + }, + "end": { + "line": 1, + "column": 18 + } + }, + "typeAnnotation": { + "type": "TSTypeReference", + "start": 17, + "end": 18, + "loc": { + "start": { + "line": 1, + "column": 17 + }, + "end": { + "line": 1, + "column": 18 + } + }, + "typeName": { + "type": "Identifier", + "start": 17, + "end": 18, + "loc": { + "start": { + "line": 1, + "column": 17 + }, + "end": { + "line": 1, + "column": 18 + }, + "identifierName": "T" + }, + "name": "T" + } + } + }, + "id": null, + "generator": false, + "async": true, + "body": { + "type": "Identifier", + "start": 22, + "end": 23, + "loc": { + "start": { + "line": 1, + "column": 22 + }, + "end": { + "line": 1, + "column": 23 + }, + "identifierName": "a" + }, + "name": "a" + } + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "async", + "start": 0, + "end": 5, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 5 + } + } + }, + { + "type": { + "label": "/<=/>=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": 8, + "updateContext": null + }, + "value": "<", + "start": 6, + "end": 7, + "loc": { + "start": { + "line": 1, + "column": 6 + }, + "end": { + "line": 1, + "column": 7 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "T", + "start": 7, + "end": 8, + "loc": { + "start": { + "line": 1, + "column": 7 + }, + "end": { + "line": 1, + "column": 8 + } + } + }, + { + "type": { + "label": "/<=/>=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": 8, + "updateContext": null + }, + "value": ">", + "start": 8, + "end": 9, + "loc": { + "start": { + "line": 1, + "column": 8 + }, + "end": { + "line": 1, + "column": 9 + } + } + }, + { + "type": { + "label": "(", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start": 9, + "end": 10, + "loc": { + "start": { + "line": 1, + "column": 9 + }, + "end": { + "line": 1, + "column": 10 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "a", + "start": 10, + "end": 11, + "loc": { + "start": { + "line": 1, + "column": 10 + }, + "end": { + "line": 1, + "column": 11 + } + } + }, + { + "type": { + "label": ":", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 11, + "end": 12, + "loc": { + "start": { + "line": 1, + "column": 11 + }, + "end": { + "line": 1, + "column": 12 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "T", + "start": 13, + "end": 14, + "loc": { + "start": { + "line": 1, + "column": 13 + }, + "end": { + "line": 1, + "column": 14 + } + } + }, + { + "type": { + "label": ")", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start": 14, + "end": 15, + "loc": { + "start": { + "line": 1, + "column": 14 + }, + "end": { + "line": 1, + "column": 15 + } + } + }, + { + "type": { + "label": ":", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 15, + "end": 16, + "loc": { + "start": { + "line": 1, + "column": 15 + }, + "end": { + "line": 1, + "column": 16 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "T", + "start": 17, + "end": 18, + "loc": { + "start": { + "line": 1, + "column": 17 + }, + "end": { + "line": 1, + "column": 18 + } + } + }, + { + "type": { + "label": "=>", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 19, + "end": 21, + "loc": { + "start": { + "line": 1, + "column": 19 + }, + "end": { + "line": 1, + "column": 21 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "a", + "start": 22, + "end": 23, + "loc": { + "start": { + "line": 1, + "column": 22 + }, + "end": { + "line": 1, + "column": 23 + } + } + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 23, + "end": 24, + "loc": { + "start": { + "line": 1, + "column": 23 + }, + "end": { + "line": 1, + "column": 24 + } + } + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 24, + "end": 24, + "loc": { + "start": { + "line": 1, + "column": 24 + }, + "end": { + "line": 1, + "column": 24 + } + } + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic/output.json b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic/output.json deleted file mode 100644 index 89af25ded2c0..000000000000 --- a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic/output.json +++ /dev/null @@ -1,231 +0,0 @@ -{ - "type": "File", - "start": 0, - "end": 24, - "loc": { - "start": { - "line": 1, - "column": 0 - }, - "end": { - "line": 1, - "column": 24 - } - }, - "program": { - "type": "Program", - "start": 0, - "end": 24, - "loc": { - "start": { - "line": 1, - "column": 0 - }, - "end": { - "line": 1, - "column": 24 - } - }, - "sourceType": "module", - "interpreter": null, - "body": [ - { - "type": "ExpressionStatement", - "start": 0, - "end": 24, - "loc": { - "start": { - "line": 1, - "column": 0 - }, - "end": { - "line": 1, - "column": 24 - } - }, - "expression": { - "type": "ArrowFunctionExpression", - "start": 0, - "end": 23, - "loc": { - "start": { - "line": 1, - "column": 0 - }, - "end": { - "line": 1, - "column": 23 - } - }, - "typeParameters": { - "type": "TSTypeParameterDeclaration", - "start": 6, - "end": 9, - "loc": { - "start": { - "line": 1, - "column": 6 - }, - "end": { - "line": 1, - "column": 9 - } - }, - "params": [ - { - "type": "TSTypeParameter", - "start": 7, - "end": 8, - "loc": { - "start": { - "line": 1, - "column": 7 - }, - "end": { - "line": 1, - "column": 8 - } - }, - "name": "T" - } - ] - }, - "params": [ - { - "type": "Identifier", - "start": 10, - "end": 14, - "loc": { - "start": { - "line": 1, - "column": 10 - }, - "end": { - "line": 1, - "column": 14 - }, - "identifierName": "a" - }, - "name": "a", - "typeAnnotation": { - "type": "TSTypeAnnotation", - "start": 11, - "end": 14, - "loc": { - "start": { - "line": 1, - "column": 11 - }, - "end": { - "line": 1, - "column": 14 - } - }, - "typeAnnotation": { - "type": "TSTypeReference", - "start": 13, - "end": 14, - "loc": { - "start": { - "line": 1, - "column": 13 - }, - "end": { - "line": 1, - "column": 14 - } - }, - "typeName": { - "type": "Identifier", - "start": 13, - "end": 14, - "loc": { - "start": { - "line": 1, - "column": 13 - }, - "end": { - "line": 1, - "column": 14 - }, - "identifierName": "T" - }, - "name": "T" - } - } - } - } - ], - "returnType": { - "type": "TSTypeAnnotation", - "start": 15, - "end": 18, - "loc": { - "start": { - "line": 1, - "column": 15 - }, - "end": { - "line": 1, - "column": 18 - } - }, - "typeAnnotation": { - "type": "TSTypeReference", - "start": 17, - "end": 18, - "loc": { - "start": { - "line": 1, - "column": 17 - }, - "end": { - "line": 1, - "column": 18 - } - }, - "typeName": { - "type": "Identifier", - "start": 17, - "end": 18, - "loc": { - "start": { - "line": 1, - "column": 17 - }, - "end": { - "line": 1, - "column": 18 - }, - "identifierName": "T" - }, - "name": "T" - } - } - }, - "id": null, - "generator": false, - "async": true, - "body": { - "type": "Identifier", - "start": 22, - "end": 23, - "loc": { - "start": { - "line": 1, - "column": 22 - }, - "end": { - "line": 1, - "column": 23 - }, - "identifierName": "a" - }, - "name": "a" - } - } - } - ], - "directives": [] - } -} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex/input.ts b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/input.ts similarity index 100% rename from packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex/input.ts rename to packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/input.ts diff --git a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/options.json b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/options.json new file mode 100644 index 000000000000..359dbf5e9514 --- /dev/null +++ b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/options.json @@ -0,0 +1,5 @@ +{ + "sourceType": "module", + "plugins": ["typescript"], + "tokens": true +} diff --git a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json new file mode 100644 index 000000000000..e22e2f6ab633 --- /dev/null +++ b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json @@ -0,0 +1,804 @@ +{ + "type": "File", + "start": 0, + "end": 52, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 52 + } + }, + "program": { + "type": "Program", + "start": 0, + "end": 52, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 52 + } + }, + "sourceType": "module", + "interpreter": null, + "body": [ + { + "type": "TSTypeAliasDeclaration", + "start": 0, + "end": 52, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 52 + } + }, + "id": { + "type": "Identifier", + "start": 5, + "end": 6, + "loc": { + "start": { + "line": 1, + "column": 5 + }, + "end": { + "line": 1, + "column": 6 + }, + "identifierName": "T" + }, + "name": "T" + }, + "typeParameters": { + "type": "TSTypeParameterDeclaration", + "start": 6, + "end": 40, + "loc": { + "start": { + "line": 1, + "column": 6 + }, + "end": { + "line": 1, + "column": 40 + } + }, + "params": [ + { + "type": "TSTypeParameter", + "start": 7, + "end": 39, + "loc": { + "start": { + "line": 1, + "column": 7 + }, + "end": { + "line": 1, + "column": 39 + } + }, + "name": "U", + "constraint": { + "type": "TSObjectKeyword", + "start": 17, + "end": 23, + "loc": { + "start": { + "line": 1, + "column": 17 + }, + "end": { + "line": 1, + "column": 23 + } + } + }, + "default": { + "type": "TSTypeLiteral", + "start": 26, + "end": 39, + "loc": { + "start": { + "line": 1, + "column": 26 + }, + "end": { + "line": 1, + "column": 39 + } + }, + "members": [ + { + "type": "TSPropertySignature", + "start": 28, + "end": 37, + "loc": { + "start": { + "line": 1, + "column": 28 + }, + "end": { + "line": 1, + "column": 37 + } + }, + "key": { + "type": "Identifier", + "start": 28, + "end": 29, + "loc": { + "start": { + "line": 1, + "column": 28 + }, + "end": { + "line": 1, + "column": 29 + }, + "identifierName": "x" + }, + "name": "x" + }, + "computed": false, + "typeAnnotation": { + "type": "TSTypeAnnotation", + "start": 29, + "end": 37, + "loc": { + "start": { + "line": 1, + "column": 29 + }, + "end": { + "line": 1, + "column": 37 + } + }, + "typeAnnotation": { + "type": "TSNumberKeyword", + "start": 31, + "end": 37, + "loc": { + "start": { + "line": 1, + "column": 31 + }, + "end": { + "line": 1, + "column": 37 + } + } + } + } + } + ] + } + } + ] + }, + "typeAnnotation": { + "type": "TSTypeReference", + "start": 43, + "end": 51, + "loc": { + "start": { + "line": 1, + "column": 43 + }, + "end": { + "line": 1, + "column": 51 + } + }, + "typeName": { + "type": "Identifier", + "start": 43, + "end": 48, + "loc": { + "start": { + "line": 1, + "column": 43 + }, + "end": { + "line": 1, + "column": 48 + }, + "identifierName": "Array" + }, + "name": "Array" + }, + "typeParameters": { + "type": "TSTypeParameterInstantiation", + "start": 48, + "end": 51, + "loc": { + "start": { + "line": 1, + "column": 48 + }, + "end": { + "line": 1, + "column": 51 + } + }, + "params": [ + { + "type": "TSTypeReference", + "start": 49, + "end": 50, + "loc": { + "start": { + "line": 1, + "column": 49 + }, + "end": { + "line": 1, + "column": 50 + } + }, + "typeName": { + "type": "Identifier", + "start": 49, + "end": 50, + "loc": { + "start": { + "line": 1, + "column": 49 + }, + "end": { + "line": 1, + "column": 50 + }, + "identifierName": "U" + }, + "name": "U" + } + } + ] + } + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "type", + "start": 0, + "end": 4, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 4 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "T", + "start": 5, + "end": 6, + "loc": { + "start": { + "line": 1, + "column": 5 + }, + "end": { + "line": 1, + "column": 6 + } + } + }, + { + "type": { + "label": "/<=/>=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": 8, + "updateContext": null + }, + "value": "<", + "start": 6, + "end": 7, + "loc": { + "start": { + "line": 1, + "column": 6 + }, + "end": { + "line": 1, + "column": 7 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "U", + "start": 7, + "end": 8, + "loc": { + "start": { + "line": 1, + "column": 7 + }, + "end": { + "line": 1, + "column": 8 + } + } + }, + { + "type": { + "label": "extends", + "keyword": "extends", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "extends", + "start": 9, + "end": 16, + "loc": { + "start": { + "line": 1, + "column": 9 + }, + "end": { + "line": 1, + "column": 16 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "object", + "start": 17, + "end": 23, + "loc": { + "start": { + "line": 1, + "column": 17 + }, + "end": { + "line": 1, + "column": 23 + } + } + }, + { + "type": { + "label": "=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": true, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "=", + "start": 24, + "end": 25, + "loc": { + "start": { + "line": 1, + "column": 24 + }, + "end": { + "line": 1, + "column": 25 + } + } + }, + { + "type": { + "label": "{", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start": 26, + "end": 27, + "loc": { + "start": { + "line": 1, + "column": 26 + }, + "end": { + "line": 1, + "column": 27 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "x", + "start": 28, + "end": 29, + "loc": { + "start": { + "line": 1, + "column": 28 + }, + "end": { + "line": 1, + "column": 29 + } + } + }, + { + "type": { + "label": ":", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 29, + "end": 30, + "loc": { + "start": { + "line": 1, + "column": 29 + }, + "end": { + "line": 1, + "column": 30 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "number", + "start": 31, + "end": 37, + "loc": { + "start": { + "line": 1, + "column": 31 + }, + "end": { + "line": 1, + "column": 37 + } + } + }, + { + "type": { + "label": "}", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start": 38, + "end": 39, + "loc": { + "start": { + "line": 1, + "column": 38 + }, + "end": { + "line": 1, + "column": 39 + } + } + }, + { + "type": { + "label": "/<=/>=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": 8, + "updateContext": null + }, + "value": ">", + "start": 39, + "end": 40, + "loc": { + "start": { + "line": 1, + "column": 39 + }, + "end": { + "line": 1, + "column": 40 + } + } + }, + { + "type": { + "label": "=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": true, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "value": "=", + "start": 41, + "end": 42, + "loc": { + "start": { + "line": 1, + "column": 41 + }, + "end": { + "line": 1, + "column": 42 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "Array", + "start": 43, + "end": 48, + "loc": { + "start": { + "line": 1, + "column": 43 + }, + "end": { + "line": 1, + "column": 48 + } + } + }, + { + "type": { + "label": "/<=/>=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": 8, + "updateContext": null + }, + "value": "<", + "start": 48, + "end": 49, + "loc": { + "start": { + "line": 1, + "column": 48 + }, + "end": { + "line": 1, + "column": 49 + } + } + }, + { + "type": { + "label": "name", + "beforeExpr": false, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "value": "U", + "start": 49, + "end": 50, + "loc": { + "start": { + "line": 1, + "column": 49 + }, + "end": { + "line": 1, + "column": 50 + } + } + }, + { + "type": { + "label": "/<=/>=", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": 8, + "updateContext": null + }, + "value": ">", + "start": 50, + "end": 51, + "loc": { + "start": { + "line": 1, + "column": 50 + }, + "end": { + "line": 1, + "column": 51 + } + } + }, + { + "type": { + "label": ";", + "beforeExpr": true, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 51, + "end": 52, + "loc": { + "start": { + "line": 1, + "column": 51 + }, + "end": { + "line": 1, + "column": 52 + } + } + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start": 52, + "end": 52, + "loc": { + "start": { + "line": 1, + "column": 52 + }, + "end": { + "line": 1, + "column": 52 + } + } + } + ] +} diff --git a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex/output.json b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex/output.json deleted file mode 100644 index 1e07768dcafd..000000000000 --- a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex/output.json +++ /dev/null @@ -1,276 +0,0 @@ -{ - "type": "File", - "start": 0, - "end": 52, - "loc": { - "start": { - "line": 1, - "column": 0 - }, - "end": { - "line": 1, - "column": 52 - } - }, - "program": { - "type": "Program", - "start": 0, - "end": 52, - "loc": { - "start": { - "line": 1, - "column": 0 - }, - "end": { - "line": 1, - "column": 52 - } - }, - "sourceType": "module", - "interpreter": null, - "body": [ - { - "type": "TSTypeAliasDeclaration", - "start": 0, - "end": 52, - "loc": { - "start": { - "line": 1, - "column": 0 - }, - "end": { - "line": 1, - "column": 52 - } - }, - "id": { - "type": "Identifier", - "start": 5, - "end": 6, - "loc": { - "start": { - "line": 1, - "column": 5 - }, - "end": { - "line": 1, - "column": 6 - }, - "identifierName": "T" - }, - "name": "T" - }, - "typeParameters": { - "type": "TSTypeParameterDeclaration", - "start": 6, - "end": 40, - "loc": { - "start": { - "line": 1, - "column": 6 - }, - "end": { - "line": 1, - "column": 40 - } - }, - "params": [ - { - "type": "TSTypeParameter", - "start": 7, - "end": 39, - "loc": { - "start": { - "line": 1, - "column": 7 - }, - "end": { - "line": 1, - "column": 39 - } - }, - "name": "U", - "constraint": { - "type": "TSObjectKeyword", - "start": 17, - "end": 23, - "loc": { - "start": { - "line": 1, - "column": 17 - }, - "end": { - "line": 1, - "column": 23 - } - } - }, - "default": { - "type": "TSTypeLiteral", - "start": 26, - "end": 39, - "loc": { - "start": { - "line": 1, - "column": 26 - }, - "end": { - "line": 1, - "column": 39 - } - }, - "members": [ - { - "type": "TSPropertySignature", - "start": 28, - "end": 37, - "loc": { - "start": { - "line": 1, - "column": 28 - }, - "end": { - "line": 1, - "column": 37 - } - }, - "key": { - "type": "Identifier", - "start": 28, - "end": 29, - "loc": { - "start": { - "line": 1, - "column": 28 - }, - "end": { - "line": 1, - "column": 29 - }, - "identifierName": "x" - }, - "name": "x" - }, - "computed": false, - "typeAnnotation": { - "type": "TSTypeAnnotation", - "start": 29, - "end": 37, - "loc": { - "start": { - "line": 1, - "column": 29 - }, - "end": { - "line": 1, - "column": 37 - } - }, - "typeAnnotation": { - "type": "TSNumberKeyword", - "start": 31, - "end": 37, - "loc": { - "start": { - "line": 1, - "column": 31 - }, - "end": { - "line": 1, - "column": 37 - } - } - } - } - } - ] - } - } - ] - }, - "typeAnnotation": { - "type": "TSTypeReference", - "start": 43, - "end": 51, - "loc": { - "start": { - "line": 1, - "column": 43 - }, - "end": { - "line": 1, - "column": 51 - } - }, - "typeName": { - "type": "Identifier", - "start": 43, - "end": 48, - "loc": { - "start": { - "line": 1, - "column": 43 - }, - "end": { - "line": 1, - "column": 48 - }, - "identifierName": "Array" - }, - "name": "Array" - }, - "typeParameters": { - "type": "TSTypeParameterInstantiation", - "start": 48, - "end": 51, - "loc": { - "start": { - "line": 1, - "column": 48 - }, - "end": { - "line": 1, - "column": 51 - } - }, - "params": [ - { - "type": "TSTypeReference", - "start": 49, - "end": 50, - "loc": { - "start": { - "line": 1, - "column": 49 - }, - "end": { - "line": 1, - "column": 50 - } - }, - "typeName": { - "type": "Identifier", - "start": 49, - "end": 50, - "loc": { - "start": { - "line": 1, - "column": 49 - }, - "end": { - "line": 1, - "column": 50 - }, - "identifierName": "U" - }, - "name": "U" - } - } - ] - } - } - } - ], - "directives": [] - } -} \ No newline at end of file From dfe64801b2a4dcae7d0408157b83020b7e94606f Mon Sep 17 00:00:00 2001 From: Chunpeng Huo Date: Sat, 18 Jan 2020 14:24:35 +1100 Subject: [PATCH 3/5] fix(babel-parser): avoid duplicated tokens trapped by mainly typescript/flow plugins --- packages/babel-parser/src/tokenizer/index.js | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index b3cd773e3bcb..1829c83378cb 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -127,13 +127,25 @@ export default class Tokenizer extends LocationParser { this.isLookahead = false; } + pushToken(token: Token | N.Comment) { + // Pop out invalid tokens trapped by try-catch parsing. + // Those parsing branches are mainly created by typescript and flow plugins. + while ( + this.tokens.length > 0 && + this.tokens[this.tokens.length - 1].end > token.start + ) { + this.tokens.pop(); + } + this.tokens.push(token); + } + // Move to the next token next(): void { if (!this.isLookahead) { this.checkKeywordEscapes(); if (this.options.tokens) { - this.tokens.push(new Token(this.state)); + this.pushToken(new Token(this.state)); } } @@ -246,7 +258,7 @@ export default class Tokenizer extends LocationParser { loc: new SourceLocation(startLoc, endLoc), }; - if (this.options.tokens) this.tokens.push(comment); + if (this.options.tokens) this.pushToken(comment); this.state.comments.push(comment); this.addComment(comment); } From cb2449e5a07b157a83bf3fa056265a673436fe73 Mon Sep 17 00:00:00 2001 From: Chunpeng Huo Date: Sun, 19 Jan 2020 00:37:08 +1100 Subject: [PATCH 4/5] test(babel-parser): update output.json to latest master result --- .../arrow-function/async-generic-tokens-true/output.json | 6 +++--- .../type-alias/generic-complex-tokens-true/output.json | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json index 5a23156c9b3b..aa657ee84f3a 100644 --- a/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/output.json @@ -265,7 +265,7 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": 8, + "binop": 7, "updateContext": null }, "value": "<", @@ -318,7 +318,7 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": 8, + "binop": 7, "updateContext": null }, "value": ">", @@ -620,4 +620,4 @@ } } ] -} \ No newline at end of file +} diff --git a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json index e22e2f6ab633..e92d91a761a3 100644 --- a/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json +++ b/packages/babel-parser/test/fixtures/typescript/type-alias/generic-complex-tokens-true/output.json @@ -336,7 +336,7 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": 8, + "binop": 7, "updateContext": null }, "value": "<", @@ -598,7 +598,7 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": 8, + "binop": 7, "updateContext": null }, "value": ">", @@ -678,7 +678,7 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": 8, + "binop": 7, "updateContext": null }, "value": "<", @@ -731,7 +731,7 @@ "isAssign": false, "prefix": false, "postfix": false, - "binop": 8, + "binop": 7, "updateContext": null }, "value": ">", From a2f8085da0f47c83523b44a58628ba0d30709746 Mon Sep 17 00:00:00 2001 From: Chunpeng Huo Date: Mon, 20 Jan 2020 11:12:19 +1100 Subject: [PATCH 5/5] chore(babel-parser): improve performance by storing tokensLength in state --- packages/babel-parser/src/tokenizer/index.js | 8 ++------ packages/babel-parser/src/tokenizer/state.js | 3 +++ 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 1829c83378cb..8558137f719f 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -130,13 +130,9 @@ export default class Tokenizer extends LocationParser { pushToken(token: Token | N.Comment) { // Pop out invalid tokens trapped by try-catch parsing. // Those parsing branches are mainly created by typescript and flow plugins. - while ( - this.tokens.length > 0 && - this.tokens[this.tokens.length - 1].end > token.start - ) { - this.tokens.pop(); - } + this.tokens.length = this.state.tokensLength; this.tokens.push(token); + ++this.state.tokensLength; } // Move to the next token diff --git a/packages/babel-parser/src/tokenizer/state.js b/packages/babel-parser/src/tokenizer/state.js index da9bb6114dff..d3a874780368 100644 --- a/packages/babel-parser/src/tokenizer/state.js +++ b/packages/babel-parser/src/tokenizer/state.js @@ -149,6 +149,9 @@ export default class State { // `export default foo;` and `export { foo as default };`. exportedIdentifiers: Array = []; + // Tokens length in token store + tokensLength: number = 0; + curPosition(): Position { return new Position(this.curLine, this.pos - this.lineStart); }