From df9ee2c7cda177b0a8810344ad7a41ab456b1b38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Hu=C3=A1ng=20J=C3=B9nli=C3=A0ng?= Date: Tue, 11 Aug 2020 13:43:03 -0400 Subject: [PATCH] Simplify tokenizer update context (#11944) * replace lineBreak.test by hasPrecedingLingBreak * refactor: simplify updateContext --- .../babel-parser/src/parser/expression.js | 24 +++++++++---------- packages/babel-parser/src/parser/statement.js | 4 +--- .../babel-parser/src/tokenizer/context.js | 15 ++---------- packages/babel-parser/src/tokenizer/index.js | 10 +++----- packages/babel-parser/src/tokenizer/types.js | 3 ++- 5 files changed, 19 insertions(+), 37 deletions(-) diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index 74d09fd8a05f..9df262f9cf13 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -2306,20 +2306,23 @@ export default class ExpressionParser extends LValParser { parseIdentifierName(pos: number, liberal?: boolean): string { let name: string; - if (this.match(tt.name)) { + const { start, type } = this.state; + + if (type === tt.name) { name = this.state.value; - } else if (this.state.type.keyword) { - name = this.state.type.keyword; + } else if (type.keyword) { + name = type.keyword; // `class` and `function` keywords push function-type token context into this.context. // But there is no chance to pop the context if the keyword is consumed // as an identifier such as a property name. - const context = this.state.context; + const curContext = this.curContext(); if ( - (name === "class" || name === "function") && - context[context.length - 1].token === "function" + (type === tt._class || type === tt._function) && + (curContext === ct.functionStatement || + curContext === ct.functionExpression) ) { - context.pop(); + this.state.context.pop(); } } else { throw this.unexpected(); @@ -2330,12 +2333,7 @@ export default class ExpressionParser extends LValParser { // This will prevent this.next() from throwing about unexpected escapes. this.state.type = tt.name; } else { - this.checkReservedWord( - name, - this.state.start, - !!this.state.type.keyword, - false, - ); + this.checkReservedWord(name, start, !!type.keyword, false); } this.next(); diff --git a/packages/babel-parser/src/parser/statement.js b/packages/babel-parser/src/parser/statement.js index ea5b295a9dab..a3e17975d1eb 100644 --- a/packages/babel-parser/src/parser/statement.js +++ b/packages/babel-parser/src/parser/statement.js @@ -629,9 +629,7 @@ export default class StatementParser extends ExpressionParser { parseThrowStatement(node: N.ThrowStatement): N.ThrowStatement { this.next(); - if ( - lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start)) - ) { + if (this.hasPrecedingLineBreak()) { this.raise(this.state.lastTokEnd, Errors.NewlineAfterThrow); } node.argument = this.parseExpression(); diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js index 8508e69bdd69..f93c19971efb 100644 --- a/packages/babel-parser/src/tokenizer/context.js +++ b/packages/babel-parser/src/tokenizer/context.js @@ -5,7 +5,6 @@ // See https://github.com/mozilla/sweet.js/wiki/design import { types as tt } from "./types"; -import { lineBreak } from "../util/whitespace"; export class TokContext { constructor( @@ -105,17 +104,11 @@ tt.incDec.updateContext = function () { }; tt._function.updateContext = tt._class.updateContext = function (prevType) { - if (prevType === tt.dot || prevType === tt.questionDot) { - // when function/class follows dot/questionDot, it is part of - // (optional)MemberExpression, then we don't need to push new token context - } else if ( + if ( prevType.beforeExpr && prevType !== tt.semi && prevType !== tt._else && - !( - prevType === tt._return && - lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start)) - ) && + !(prevType === tt._return && this.hasPrecedingLineBreak()) && !( (prevType === tt.colon || prevType === tt.braceL) && this.curContext() === types.b_stat @@ -138,10 +131,6 @@ tt.backQuote.updateContext = function () { this.state.exprAllowed = false; }; -tt.star.updateContext = function () { - this.state.exprAllowed = false; -}; - // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR tt.braceHashL.updateContext = function () { this.state.context.push(types.recordExpression); diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index d8104c4f9b45..cbc0efab0fe8 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -114,6 +114,7 @@ export default class Tokenizer extends ParserErrors { // Forward-declarations // parser/util.js /*:: + +hasPrecedingLineBreak: () => boolean; +unexpected: (pos?: ?number, messageOrType?: string | TokenType) => empty; +expectPlugin: (name: string, pos?: ?number) => true; */ @@ -603,10 +604,7 @@ export default class Tokenizer extends ParserErrors { next === charCodes.dash && !this.inModule && this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan && - (this.state.lastTokEnd === 0 || - lineBreak.test( - this.input.slice(this.state.lastTokEnd, this.state.pos), - )) + (this.state.lastTokEnd === 0 || this.hasPrecedingLineBreak()) ) { // A `-->` line comment this.skipLineComment(3); @@ -1525,9 +1523,7 @@ export default class Tokenizer extends ParserErrors { prevType === tt._return || (prevType === tt.name && this.state.exprAllowed) ) { - return lineBreak.test( - this.input.slice(this.state.lastTokEnd, this.state.start), - ); + return this.hasPrecedingLineBreak(); } if ( diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js index 535e16bd1204..ca67cb0732cf 100644 --- a/packages/babel-parser/src/tokenizer/types.js +++ b/packages/babel-parser/src/tokenizer/types.js @@ -155,7 +155,8 @@ export const types: { [name: string]: TokenType } = { plusMin: new TokenType("+/-", { beforeExpr, binop: 9, prefix, startsExpr }), // startsExpr: required by v8intrinsic plugin modulo: new TokenType("%", { beforeExpr, binop: 10, startsExpr }), - star: createBinop("*", 10), + // unset `beforeExpr` as it can be `function *` + star: new TokenType("*", { binop: 10 }), slash: createBinop("/", 10), exponent: new TokenType("**", { beforeExpr,