Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Simplify tokenizer update context #11944

Merged
merged 2 commits into from Aug 11, 2020
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
24 changes: 11 additions & 13 deletions packages/babel-parser/src/parser/expression.js
Expand Up @@ -2306,20 +2306,23 @@ export default class ExpressionParser extends LValParser {
parseIdentifierName(pos: number, liberal?: boolean): string {
let name: string;

if (this.match(tt.name)) {
const { start, type } = this.state;

if (type === tt.name) {
name = this.state.value;
} else if (this.state.type.keyword) {
name = this.state.type.keyword;
} else if (type.keyword) {
name = type.keyword;

// `class` and `function` keywords push function-type token context into this.context.
// But there is no chance to pop the context if the keyword is consumed
// as an identifier such as a property name.
const context = this.state.context;
const curContext = this.curContext();
if (
(name === "class" || name === "function") &&
context[context.length - 1].token === "function"
(type === tt._class || type === tt._function) &&
(curContext === ct.functionStatement ||
curContext === ct.functionExpression)
) {
context.pop();
this.state.context.pop();
}
} else {
throw this.unexpected();
Expand All @@ -2330,12 +2333,7 @@ export default class ExpressionParser extends LValParser {
// This will prevent this.next() from throwing about unexpected escapes.
this.state.type = tt.name;
} else {
this.checkReservedWord(
name,
this.state.start,
!!this.state.type.keyword,
false,
);
this.checkReservedWord(name, start, !!type.keyword, false);
}

this.next();
Expand Down
4 changes: 1 addition & 3 deletions packages/babel-parser/src/parser/statement.js
Expand Up @@ -629,9 +629,7 @@ export default class StatementParser extends ExpressionParser {

parseThrowStatement(node: N.ThrowStatement): N.ThrowStatement {
this.next();
if (
lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start))
) {
if (this.hasPrecedingLineBreak()) {
this.raise(this.state.lastTokEnd, Errors.NewlineAfterThrow);
}
node.argument = this.parseExpression();
Expand Down
15 changes: 2 additions & 13 deletions packages/babel-parser/src/tokenizer/context.js
Expand Up @@ -5,7 +5,6 @@
// See https://github.com/mozilla/sweet.js/wiki/design

import { types as tt } from "./types";
import { lineBreak } from "../util/whitespace";

export class TokContext {
constructor(
Expand Down Expand Up @@ -105,17 +104,11 @@ tt.incDec.updateContext = function () {
};

tt._function.updateContext = tt._class.updateContext = function (prevType) {
if (prevType === tt.dot || prevType === tt.questionDot) {
// when function/class follows dot/questionDot, it is part of
// (optional)MemberExpression, then we don't need to push new token context
} else if (
if (
prevType.beforeExpr &&
prevType !== tt.semi &&
prevType !== tt._else &&
!(
prevType === tt._return &&
lineBreak.test(this.input.slice(this.state.lastTokEnd, this.state.start))
) &&
!(prevType === tt._return && this.hasPrecedingLineBreak()) &&
!(
(prevType === tt.colon || prevType === tt.braceL) &&
this.curContext() === types.b_stat
Expand All @@ -138,10 +131,6 @@ tt.backQuote.updateContext = function () {
this.state.exprAllowed = false;
};

tt.star.updateContext = function () {
this.state.exprAllowed = false;
};

// we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR
tt.braceHashL.updateContext = function () {
this.state.context.push(types.recordExpression);
Expand Down
10 changes: 3 additions & 7 deletions packages/babel-parser/src/tokenizer/index.js
Expand Up @@ -114,6 +114,7 @@ export default class Tokenizer extends ParserErrors {
// Forward-declarations
// parser/util.js
/*::
+hasPrecedingLineBreak: () => boolean;
+unexpected: (pos?: ?number, messageOrType?: string | TokenType) => empty;
+expectPlugin: (name: string, pos?: ?number) => true;
*/
Expand Down Expand Up @@ -603,10 +604,7 @@ export default class Tokenizer extends ParserErrors {
next === charCodes.dash &&
!this.inModule &&
this.input.charCodeAt(this.state.pos + 2) === charCodes.greaterThan &&
(this.state.lastTokEnd === 0 ||
lineBreak.test(
this.input.slice(this.state.lastTokEnd, this.state.pos),
))
(this.state.lastTokEnd === 0 || this.hasPrecedingLineBreak())
) {
// A `-->` line comment
this.skipLineComment(3);
Expand Down Expand Up @@ -1525,9 +1523,7 @@ export default class Tokenizer extends ParserErrors {
prevType === tt._return ||
(prevType === tt.name && this.state.exprAllowed)
) {
return lineBreak.test(
this.input.slice(this.state.lastTokEnd, this.state.start),
);
return this.hasPrecedingLineBreak();
}

if (
Expand Down
3 changes: 2 additions & 1 deletion packages/babel-parser/src/tokenizer/types.js
Expand Up @@ -155,7 +155,8 @@ export const types: { [name: string]: TokenType } = {
plusMin: new TokenType("+/-", { beforeExpr, binop: 9, prefix, startsExpr }),
// startsExpr: required by v8intrinsic plugin
modulo: new TokenType("%", { beforeExpr, binop: 10, startsExpr }),
star: createBinop("*", 10),
// unset `beforeExpr` as it can be `function *`
star: new TokenType("*", { binop: 10 }),
Copy link
Contributor Author

@JLHwung JLHwung Aug 10, 2020

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

So we can remove

tt.star.updateContext = function () {
  this.state.exprAllowed = false;
};

because exprAllowed is always implied from beforeExpr if there isn't a customized updateContext handler.

slash: createBinop("/", 10),
exponent: new TokenType("**", {
beforeExpr,
Expand Down