diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js index bf47b980ba1a..812fa605a6af 100644 --- a/packages/babel-parser/src/parser/expression.js +++ b/packages/babel-parser/src/parser/expression.js @@ -613,7 +613,7 @@ export default class ExpressionParser extends LValParser { } else if (this.match(tt.questionDot)) { this.expectPlugin("optionalChaining"); state.optionalChainMember = true; - if (noCalls && this.lookahead().type === tt.parenL) { + if (noCalls && this.lookaheadCharCode() === charCodes.leftParenthesis) { state.stop = true; return base; } diff --git a/packages/babel-parser/src/parser/statement.js b/packages/babel-parser/src/parser/statement.js index bfadb601dbcf..8ce0c9d64418 100644 --- a/packages/babel-parser/src/parser/statement.js +++ b/packages/babel-parser/src/parser/statement.js @@ -8,7 +8,7 @@ import { isIdentifierStart, keywordRelationalOperator, } from "../util/identifier"; -import { lineBreak, skipWhiteSpace } from "../util/whitespace"; +import { lineBreak } from "../util/whitespace"; import * as charCodes from "charcodes"; import { BIND_CLASS, @@ -105,10 +105,7 @@ export default class StatementParser extends ExpressionParser { if (!this.isContextual("let")) { return false; } - skipWhiteSpace.lastIndex = this.state.pos; - const skip = skipWhiteSpace.exec(this.input); - // $FlowIgnore - const next = this.state.pos + skip[0].length; + const next = this.nextTokenStart(); const nextCh = this.input.charCodeAt(next); // For ambiguous cases, determine if a LexicalDeclaration (or only a // Statement) is allowed here. If context is not empty then only a Statement @@ -170,7 +167,7 @@ export default class StatementParser extends ExpressionParser { case tt._for: return this.parseForStatement(node); case tt._function: - if (this.lookahead().type === tt.dot) break; + if (this.lookaheadCharCode() === charCodes.dot) break; if (context) { if (this.state.strict) { this.raise( @@ -223,8 +220,11 @@ export default class StatementParser extends ExpressionParser { return this.parseEmptyStatement(node); case tt._export: case tt._import: { - const nextToken = this.lookahead(); - if (nextToken.type === tt.parenL || nextToken.type === tt.dot) { + const nextTokenCharCode = this.lookaheadCharCode(); + if ( + nextTokenCharCode === charCodes.leftParenthesis || + nextTokenCharCode === charCodes.dot + ) { break; } @@ -1746,11 +1746,11 @@ export default class StatementParser extends ExpressionParser { maybeParseExportDeclaration(node: N.Node): boolean { if (this.shouldParseExportDeclaration()) { if (this.isContextual("async")) { - const next = this.lookahead(); + const next = this.nextTokenStart(); // export async; - if (next.type !== tt._function) { - this.unexpected(next.start, `Unexpected token, expected "function"`); + if (!this.isUnparsedContextual(next, "function")) { + this.unexpected(next, `Unexpected token, expected "function"`); } } @@ -1765,21 +1765,10 @@ export default class StatementParser extends ExpressionParser { isAsyncFunction(): boolean { if (!this.isContextual("async")) return false; - - const { pos } = this.state; - - skipWhiteSpace.lastIndex = pos; - const skip = skipWhiteSpace.exec(this.input); - - if (!skip || !skip.length) return false; - - const next = pos + skip[0].length; - + const next = this.nextTokenStart(); return ( - !lineBreak.test(this.input.slice(pos, next)) && - this.input.slice(next, next + 8) === "function" && - (next + 8 === this.length || - !isIdentifierChar(this.input.charCodeAt(next + 8))) + !lineBreak.test(this.input.slice(this.state.pos, next)) && + this.isUnparsedContextual(next, "function") ); } @@ -1841,10 +1830,10 @@ export default class StatementParser extends ExpressionParser { return false; } - const lookahead = this.lookahead(); + const next = this.nextTokenStart(); return ( - lookahead.type === tt.comma || - (lookahead.type === tt.name && lookahead.value === "from") + this.input.charCodeAt(next) === charCodes.comma || + this.isUnparsedContextual(next, "from") ); } diff --git a/packages/babel-parser/src/parser/util.js b/packages/babel-parser/src/parser/util.js index 41e19c1a3c2e..d66671551ee2 100644 --- a/packages/babel-parser/src/parser/util.js +++ b/packages/babel-parser/src/parser/util.js @@ -4,6 +4,8 @@ import { types as tt, type TokenType } from "../tokenizer/types"; import Tokenizer from "../tokenizer"; import type { Node } from "../types"; import { lineBreak, skipWhiteSpace } from "../util/whitespace"; +import { isIdentifierChar } from "../util/identifier"; +import * as charCodes from "charcodes"; const literal = /^('|")((?:\\?.)*?)\1/; @@ -26,8 +28,15 @@ export default class UtilParser extends Tokenizer { } isLookaheadRelational(op: "<" | ">"): boolean { - const l = this.lookahead(); - return l.type === tt.relational && l.value === op; + const next = this.nextTokenStart(); + if (this.input.charAt(next) === op) { + if (next + 1 === this.input.length) { + return true; + } + const afterNext = this.input.charCodeAt(next + 1); + return afterNext !== op.charCodeAt(0) && afterNext !== charCodes.equalsTo; + } + return false; } // TODO @@ -60,9 +69,18 @@ export default class UtilParser extends Tokenizer { ); } + isUnparsedContextual(nameStart: number, name: string): boolean { + const nameEnd = nameStart + name.length; + return ( + this.input.slice(nameStart, nameEnd) === name && + (nameEnd === this.input.length || + !isIdentifierChar(this.input.charCodeAt(nameEnd))) + ); + } + isLookaheadContextual(name: string): boolean { - const l = this.lookahead(); - return l.type === tt.name && l.value === name; + const next = this.nextTokenStart(); + return this.isUnparsedContextual(next, name); } // Consumes contextual keyword if possible. diff --git a/packages/babel-parser/src/plugins/typescript/index.js b/packages/babel-parser/src/plugins/typescript/index.js index 3ac78d17474a..1eedd15b83c4 100644 --- a/packages/babel-parser/src/plugins/typescript/index.js +++ b/packages/babel-parser/src/plugins/typescript/index.js @@ -19,6 +19,7 @@ import { BIND_CLASS, } from "../../util/scopeflags"; import TypeScriptScopeHandler from "./scope"; +import * as charCodes from "charcodes"; type TsModifier = | "readonly" @@ -657,7 +658,10 @@ export default (superClass: Class): Class => : this.match(tt._null) ? "TSNullKeyword" : keywordTypeFromName(this.state.value); - if (type !== undefined && this.lookahead().type !== tt.dot) { + if ( + type !== undefined && + this.lookaheadCharCode() !== charCodes.dot + ) { const node: N.TsKeywordType = this.startNode(); this.next(); return this.finishNode(node, type); @@ -1203,7 +1207,8 @@ export default (superClass: Class): Class => tsIsExternalModuleReference(): boolean { return ( - this.isContextual("require") && this.lookahead().type === tt.parenL + this.isContextual("require") && + this.lookaheadCharCode() === charCodes.leftParenthesis ); } diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index 3fdced8aaffd..1d2ea02212fd 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -13,6 +13,7 @@ import { lineBreakG, isNewLine, isWhitespace, + skipWhiteSpace, } from "../util/whitespace"; import State from "./state"; @@ -168,6 +169,18 @@ export default class Tokenizer extends LocationParser { return curr; } + nextTokenStart(): number { + const thisTokEnd = this.state.pos; + skipWhiteSpace.lastIndex = thisTokEnd; + const skip = skipWhiteSpace.exec(this.input); + // $FlowIgnore: The skipWhiteSpace ensures to match any string + return thisTokEnd + skip[0].length; + } + + lookaheadCharCode(): number { + return this.input.charCodeAt(this.nextTokenStart()); + } + // Toggle strict mode. Re-reads the next number or string to please // pedantic tests (`"use strict"; 010;` should fail). @@ -267,13 +280,7 @@ export default class Tokenizer extends LocationParser { const startLoc = this.state.curPosition(); let ch = this.input.charCodeAt((this.state.pos += startSkip)); if (this.state.pos < this.length) { - while ( - ch !== charCodes.lineFeed && - ch !== charCodes.carriageReturn && - ch !== charCodes.lineSeparator && - ch !== charCodes.paragraphSeparator && - ++this.state.pos < this.length - ) { + while (!isNewLine(ch) && ++this.state.pos < this.length) { ch = this.input.charCodeAt(this.state.pos); } } @@ -441,13 +448,7 @@ export default class Tokenizer extends LocationParser { let ch = this.input.charCodeAt(this.state.pos); if (ch !== charCodes.exclamationMark) return false; - while ( - ch !== charCodes.lineFeed && - ch !== charCodes.carriageReturn && - ch !== charCodes.lineSeparator && - ch !== charCodes.paragraphSeparator && - ++this.state.pos < this.length - ) { + while (!isNewLine(ch) && ++this.state.pos < this.length) { ch = this.input.charCodeAt(this.state.pos); }