fix(babel-parser): avoid state.clone() to clone the whole token store

packages/babel-parser/src/parser/statement.js

@@ -67,7 +67,7 @@ export default class StatementParser extends ExpressionParser {
     file.program = this.finishNode(program, "Program");
     file.comments = this.state.comments;
 
-    if (this.options.tokens) file.tokens = this.state.tokens;
+    if (this.options.tokens) file.tokens = this.tokens;
 
     return this.finishNode(file, "File");
   }

packages/babel-parser/src/tokenizer/index.js

@@ -1,6 +1,7 @@
 // @flow
 
 import type { Options } from "../options";
+import * as N from "../types";
 import type { Position } from "../util/location";
 import * as charCodes from "charcodes";
 import { isIdentifierStart, isIdentifierChar } from "../util/identifier";
@@ -114,6 +115,9 @@ export default class Tokenizer extends LocationParser {
 
   isLookahead: boolean;
 
+  // Token store.
+  tokens: Array<Token | N.Comment> = [];
+
   constructor(options: Options, input: string) {
     super();
     this.state = new State();
@@ -123,13 +127,25 @@ export default class Tokenizer extends LocationParser {
     this.isLookahead = false;
   }
 
+  pushToken(token: Token | N.Comment) {
+    // Pop out invalid tokens trapped by try-catch parsing.
+    // Those parsing branches are mainly created by typescript and flow plugins.
+    while (
+      this.tokens.length > 0 &&
+      this.tokens[this.tokens.length - 1].end > token.start
+    ) {
+      this.tokens.pop();
+    }
+    this.tokens.push(token);
+  }
+
   // Move to the next token
 
   next(): void {
     if (!this.isLookahead) {
       this.checkKeywordEscapes();
       if (this.options.tokens) {
-        this.state.tokens.push(new Token(this.state));
+        this.pushToken(new Token(this.state));
       }
     }
 
@@ -242,7 +258,7 @@ export default class Tokenizer extends LocationParser {
       loc: new SourceLocation(startLoc, endLoc),
     };
 
-    if (this.options.tokens) this.state.tokens.push(comment);
+    if (this.options.tokens) this.pushToken(comment);
     this.state.comments.push(comment);
     this.addComment(comment);
   }

packages/babel-parser/src/tokenizer/state.js

@@ -5,7 +5,6 @@ import * as N from "../types";
 import { Position } from "../util/location";
 
 import { types as ct, type TokContext } from "./context";
-import type { Token } from "./index";
 import { types as tt, type TokenType } from "./types";
 
 type TopicContextState = {
@@ -93,9 +92,6 @@ export default class State {
   yieldPos: number = -1;
   awaitPos: number = -1;
 
-  // Token store.
-  tokens: Array<Token | N.Comment> = [];
-
   // Comment store.
   comments: Array<N.Comment> = [];
 

packages/babel-parser/test/fixtures/typescript/arrow-function/async-generic-tokens-true/options.json

@@ -0,0 +1,5 @@
+{
+  "sourceType": "module",
+  "plugins": ["typescript"],
+  "tokens": true
+}