Port babel-parser changes from 2020-04-12 to 2020-07-22 (#556)

a466f 5 months ago fix: report missing plugins on type exports (#11417)
🚫 Looks like this is already covered by the existing isTS and isFlow cases

fba64 5 months ago fix: disallow expression after binding identifier `of` (#11355)
🚫 Sucrase doesn't use token contexts.

40c51 5 months ago Set exprAllowed to false for star token (#11449)
🚫 Sucrase doesn't use token contexts.

2e4f18 5 months ago Add some parser missing plugins errors (#11478)
🚫 Validation related to babel plugins only.

fa98a 5 months ago docs: update AST spec (#11492)
🚫 Docs update

9c284 5 months ago (tag: v7.9.6) v7.9.6
🚫 Release only

90a91 5 months ago Update Flow to 0.123.0 (#11500)
🚫 Flow version bump with one change to the tokenizer that isn't present in sucrase

31b36 5 months ago Use ?. where it represents the intended semantics (#11512)
🚫 Refactor to make use of ?. in the all of babel, separate effort in sucrase if needed

2f31e 4 months ago fix: allow bigInt in method name and TSLiteralType (#11547)
✅ Skipped refactor of isLiteralPropertyName in utils, but ported bigint type literal support + added test

62e68 4 months ago Fix comments for smartPipeline topic-forbidding contexts (#11597)
🚫 Comment update related to contexts

5dd7f 4 months ago Enable `import.meta` by default in `@babel/parser` (#11406)
🚫 Import meta is already always parsed

66b86 4 months ago added basic support for module attributes and tests updated (#10962)
🚫 Skipped as this is the implementation of a stage 1 proposal - the proposal is now stage 3 though, but is perhaps a separate effort? https://github.com/tc39/proposal-import-assertions

74590 4 months ago Add private-property-in-object support (#11372)
✅ Ported

bda75 4 months ago Handle private access chained on an optional chain (#11248)
✅ Ported

5da24 4 months ago (tag: v7.10.0) v7.10.0
🚫 Release only

242d9 4 months ago Use `repository.directory` field in `package.json` files (#11625)
🚫 Just babel package.json stuff

88f57 4 months ago (tag: v7.10.1) v7.10.1
🚫 Release only

b5c4a 4 months ago refactor: split locationParser into ParserErrors and error message (#11653)
🚫 Skipped, error message related refactoring

b0350 4 months ago (tag: v7.10.2) v7.10.2
🚫 Release only

71d352 4 months ago Properly parse `export default from` when `exportDefaultFrom` is not enabled (#11676)
✅ Ported the change, but was likely working already.

41085 4 months ago Update prettier to v2 (#11579)
🚫 Skipped, formatting change

b27ab 3 months ago fix: add optional: false to MemberExpression (#11709)
🚫 estree not supported.

e15a5 3 months ago Fix innercomments (#11697)
🚫 Sucrase skips comments

eea15 3 months ago Migrate from "master" branch to "main" (#11715)
🚫 Branch rename

2787e 3 months ago (tag: v7.10.3) v7.10.3
🚫 Release only

30835 3 months ago fix: implement early errors for record and tuple (#11652)
🚫 Error handling only

beca7 3 months ago Add better parser error when using jsx (#11722)
🚫 Error handling only

75c23 3 months ago Add @babel/eslint-plugin-development-internal (#11376)
🚫 Linter changes only

7fd40d 3 months ago (tag: v7.10.4) v7.10.4
🚫 Release only

b1b21 3 months ago docs: add AST spec on optional chain [skip ci] (#11729)
🚫 Documentation change only

d6762 3 months ago fix: throw expect jsx plugin error when an idStart or > is seen (#11774)
🚫 Error handling only

02c8f 3 months ago fix: add optional: false to chained optional call expression (#11814)
🚫 Sets node.optional, this can be skipped since optional chaining is handled differently in Sucrase

f7964 2 months ago (tag: v7.10.5) v7.10.5
🚫 Release only

8f191 10 weeks ago chore: fix typo in codebase (#11846)
🚫 These typos aren't present in Sucrase

f4eeff 10 weeks ago fix: correctly set innerEndPos in CoverParenthesizedExpressionAndArrowParameterList (#11847)
🚫 Fix for cone that's not present in Sucrase

3680f 9 weeks ago fix: allow 09.1_1 and 09e1_1 in sloppy mode (#11854)
🚫 Sucrase doesn't have separate handling of octals so this was not needed

2bf38 9 weeks ago fix: disallow \8, \9 in strict mode string (#11852)
🚫 Error handling only

059e91 8 weeks ago Add decimal parsing support (#11640)
✅ Ported

d7347f 8 weeks ago eslint-parser: ES2020 features (#11815)
🚫 estree not supported.

5b4b3 3 months ago TypeScript 4.0: Allow spread in the middle of tuples (#11753)
✅ Added tests, but the change did not need to be ported

8a1d7e4 3 months ago Allow unknown/any in TS catch clause param (#11755)
✅ Ported + added TypeScript tests, only change needed to support catch clause params

9e666 3 months ago Follow-up on initial TS4 catch param support (#11767)
🚫 Skipped, this detail is not included in the TypeScript parser in sucrase

eba4c 2 months ago TypeScript 4.0: Support labeled tuple elements (#11754)
✅ Ported, excluding validation logic

0e985 9 weeks ago feat: enable numericSeparator parsing support (#11863)
🚫 Skipped, already supported

b651a 9 weeks ago Enable logical assignment by default in @babel/parser (#11860) (#11869)
🚫 Skipped, already supported

benchmark/sample/expression.ts

@@ -408,6 +408,7 @@ export function parseExprAtom(): boolean {
     case tt.regexp:
     case tt.num:
     case tt.bigint:
+    case tt.decimal:
     case tt.string:
     case tt._null:
     case tt._true:

generator/generateTokenTypes.ts

@@ -48,6 +48,7 @@ class BinopTokenType extends TokenType {
 const types = {
   num: new TokenType("num"),
   bigint: new TokenType("bigint"),
+  decimal: new TokenType("decimal"),
   regexp: new TokenType("regexp"),
   string: new TokenType("string"),
   name: new TokenType("name"),

src/parser/plugins/typescript.ts

@@ -367,6 +367,7 @@ function tsParseMappedType(): void {
 function tsParseTupleType(): void {
   expect(tt.bracketL);
   while (!eat(tt.bracketR) && !state.error) {
+    // Do not validate presence of either none or only labeled elements
     tsParseTupleElementType();
     eat(tt.comma);
   }
@@ -376,11 +377,17 @@ function tsParseTupleElementType(): void {
   // parses `...TsType[]`
   if (eat(tt.ellipsis)) {
     tsParseType();
-    return;
+  } else {
+    // parses `TsType?`
+    tsParseType();
+    eat(tt.question);
+  }
+
+  // The type we parsed above was actually a label
+  if (eat(tt.colon)) {
+    // Labeled tuple types must affix the label with `...` or `?`, so no need to handle those here
+    tsParseType();
   }
-  // parses `TsType?`
-  tsParseType();
-  eat(tt.question);
 }
 
 function tsParseParenthesizedType(): void {
@@ -412,6 +419,8 @@ function tsParseNonArrayType(): void {
       return;
     case tt.string:
     case tt.num:
+    case tt.bigint:
+    case tt.decimal:
     case tt._true:
     case tt._false:
       parseLiteral();

src/parser/tokenizer/index.ts

@@ -4,7 +4,7 @@ import {input, isFlowEnabled, state} from "../traverser/base";
 import {unexpected} from "../traverser/util";
 import {charCodes} from "../util/charcodes";
 import {IS_IDENTIFIER_CHAR, IS_IDENTIFIER_START} from "../util/identifier";
-import {IS_WHITESPACE} from "../util/whitespace";
+import {IS_WHITESPACE, skipWhiteSpace} from "../util/whitespace";
 import {ContextualKeyword} from "./keywords";
 import readWord from "./readWord";
 import {TokenType, TokenType as tt} from "./types";
@@ -218,6 +218,20 @@ export function lookaheadTypeAndKeyword(): TypeAndKeyword {
   return new TypeAndKeyword(type, contextualKeyword);
 }
 
+export function nextTokenStart(): number {
+  return nextTokenStartSince(state.pos);
+}
+
+export function nextTokenStartSince(pos: number): number {
+  skipWhiteSpace.lastIndex = pos;
+  const skip = skipWhiteSpace.exec(input);
+  return pos + skip![0].length;
+}
+
+export function lookaheadCharCode(): number {
+  return input.charCodeAt(nextTokenStart());
+}
+
 // Read a single token, updating the parser object's token-related
 // properties.
 export function nextToken(): void {
@@ -749,13 +763,17 @@ function readInt(): void {
 
 function readRadixNumber(): void {
   let isBigInt = false;
+  const start = state.pos;
 
   state.pos += 2; // 0x
   readInt();
 
-  if (input.charCodeAt(state.pos) === charCodes.lowercaseN) {
+  const nextChar = input.charCodeAt(state.pos);
+  if (nextChar === charCodes.lowercaseN) {
     ++state.pos;
     isBigInt = true;
+  } else if (nextChar === charCodes.lowercaseM) {
+    unexpected("Invalid decimal", start);
   }
 
   if (isBigInt) {
@@ -769,6 +787,7 @@ function readRadixNumber(): void {
 // Read an integer, octal integer, or floating-point number.
 function readNumber(startsWithDot: boolean): void {
   let isBigInt = false;
+  let isDecimal = false;
 
   if (!startsWithDot) {
     readInt();
@@ -793,12 +812,21 @@ function readNumber(startsWithDot: boolean): void {
   if (nextChar === charCodes.lowercaseN) {
     ++state.pos;
     isBigInt = true;
+  } else if (nextChar === charCodes.lowercaseM) {
+    ++state.pos;
+    isDecimal = true;
   }
 
   if (isBigInt) {
     finishToken(tt.bigint);
     return;
   }
+
+  if (isDecimal) {
+    finishToken(tt.decimal);
+    return;
+  }
+
   finishToken(tt.num);
 }
 

src/parser/tokenizer/types.ts

@@ -13,120 +13,123 @@ export enum TokenType {
 
   num = 0, // num
   bigint = 512, // bigint
-  regexp = 1024, // regexp
-  string = 1536, // string
-  name = 2048, // name
-  eof = 2560, // eof
-  bracketL = 3072, // [
-  bracketR = 3584, // ]
-  braceL = 4096, // {
-  braceBarL = 4608, // {|
-  braceR = 5120, // }
-  braceBarR = 5632, // |}
-  parenL = 6144, // (
-  parenR = 6656, // )
-  comma = 7168, // ,
-  semi = 7680, // ;
-  colon = 8192, // :
-  doubleColon = 8704, // ::
-  dot = 9216, // .
-  question = 9728, // ?
-  questionDot = 10240, // ?.
-  arrow = 10752, // =>
-  template = 11264, // template
-  ellipsis = 11776, // ...
-  backQuote = 12288, // `
-  dollarBraceL = 12800, // ${
-  at = 13312, // @
-  hash = 13824, // #
-  eq = 14368, // = isAssign
-  assign = 14880, // _= isAssign
-  preIncDec = 15744, // ++/-- prefix postfix
-  postIncDec = 16256, // ++/-- prefix postfix
-  bang = 16512, // ! prefix
-  tilde = 17024, // ~ prefix
-  pipeline = 17409, // |> prec:1
-  nullishCoalescing = 17922, // ?? prec:2
-  logicalOR = 18434, // || prec:2
-  logicalAND = 18947, // && prec:3
-  bitwiseOR = 19460, // | prec:4
-  bitwiseXOR = 19973, // ^ prec:5
-  bitwiseAND = 20486, // & prec:6
-  equality = 20999, // ==/!= prec:7
-  lessThan = 21512, // < prec:8
-  greaterThan = 22024, // > prec:8
-  relationalOrEqual = 22536, // <=/>= prec:8
-  bitShift = 23049, // <</>> prec:9
-  plus = 23690, // + prec:10 prefix
-  minus = 24202, // - prec:10 prefix
-  modulo = 24587, // % prec:11
-  star = 25099, // * prec:11
-  slash = 25611, // / prec:11
-  exponent = 26188, // ** prec:12 rightAssociative
-  jsxName = 26624, // jsxName
-  jsxText = 27136, // jsxText
-  jsxTagStart = 27648, // jsxTagStart
-  jsxTagEnd = 28160, // jsxTagEnd
-  typeParameterStart = 28672, // typeParameterStart
-  nonNullAssertion = 29184, // nonNullAssertion
-  _break = 29712, // break keyword
-  _case = 30224, // case keyword
-  _catch = 30736, // catch keyword
-  _continue = 31248, // continue keyword
-  _debugger = 31760, // debugger keyword
-  _default = 32272, // default keyword
-  _do = 32784, // do keyword
-  _else = 33296, // else keyword
-  _finally = 33808, // finally keyword
-  _for = 34320, // for keyword
-  _function = 34832, // function keyword
-  _if = 35344, // if keyword
-  _return = 35856, // return keyword
-  _switch = 36368, // switch keyword
-  _throw = 37008, // throw keyword prefix
-  _try = 37392, // try keyword
-  _var = 37904, // var keyword
-  _let = 38416, // let keyword
-  _const = 38928, // const keyword
-  _while = 39440, // while keyword
-  _with = 39952, // with keyword
-  _new = 40464, // new keyword
-  _this = 40976, // this keyword
-  _super = 41488, // super keyword
-  _class = 42000, // class keyword
-  _extends = 42512, // extends keyword
-  _export = 43024, // export keyword
-  _import = 43536, // import keyword
-  _yield = 44048, // yield keyword
-  _null = 44560, // null keyword
-  _true = 45072, // true keyword
-  _false = 45584, // false keyword
-  _in = 46104, // in prec:8 keyword
-  _instanceof = 46616, // instanceof prec:8 keyword
-  _typeof = 47248, // typeof keyword prefix
-  _void = 47760, // void keyword prefix
-  _delete = 48272, // delete keyword prefix
-  _async = 48656, // async keyword
-  _get = 49168, // get keyword
-  _set = 49680, // set keyword
-  _declare = 50192, // declare keyword
-  _readonly = 50704, // readonly keyword
-  _abstract = 51216, // abstract keyword
-  _static = 51728, // static keyword
-  _public = 52240, // public keyword
-  _private = 52752, // private keyword
-  _protected = 53264, // protected keyword
-  _as = 53776, // as keyword
-  _enum = 54288, // enum keyword
-  _type = 54800, // type keyword
-  _implements = 55312, // implements keyword
+  decimal = 1024, // decimal
+  regexp = 1536, // regexp
+  string = 2048, // string
+  name = 2560, // name
+  eof = 3072, // eof
+  bracketL = 3584, // [
+  bracketR = 4096, // ]
+  braceL = 4608, // {
+  braceBarL = 5120, // {|
+  braceR = 5632, // }
+  braceBarR = 6144, // |}
+  parenL = 6656, // (
+  parenR = 7168, // )
+  comma = 7680, // ,
+  semi = 8192, // ;
+  colon = 8704, // :
+  doubleColon = 9216, // ::
+  dot = 9728, // .
+  question = 10240, // ?
+  questionDot = 10752, // ?.
+  arrow = 11264, // =>
+  template = 11776, // template
+  ellipsis = 12288, // ...
+  backQuote = 12800, // `
+  dollarBraceL = 13312, // ${
+  at = 13824, // @
+  hash = 14336, // #
+  eq = 14880, // = isAssign
+  assign = 15392, // _= isAssign
+  preIncDec = 16256, // ++/-- prefix postfix
+  postIncDec = 16768, // ++/-- prefix postfix
+  bang = 17024, // ! prefix
+  tilde = 17536, // ~ prefix
+  pipeline = 17921, // |> prec:1
+  nullishCoalescing = 18434, // ?? prec:2
+  logicalOR = 18946, // || prec:2
+  logicalAND = 19459, // && prec:3
+  bitwiseOR = 19972, // | prec:4
+  bitwiseXOR = 20485, // ^ prec:5
+  bitwiseAND = 20998, // & prec:6
+  equality = 21511, // ==/!= prec:7
+  lessThan = 22024, // < prec:8
+  greaterThan = 22536, // > prec:8
+  relationalOrEqual = 23048, // <=/>= prec:8
+  bitShift = 23561, // <</>> prec:9
+  plus = 24202, // + prec:10 prefix
+  minus = 24714, // - prec:10 prefix
+  modulo = 25099, // % prec:11
+  star = 25611, // * prec:11
+  slash = 26123, // / prec:11
+  exponent = 26700, // ** prec:12 rightAssociative
+  jsxName = 27136, // jsxName
+  jsxText = 27648, // jsxText
+  jsxTagStart = 28160, // jsxTagStart
+  jsxTagEnd = 28672, // jsxTagEnd
+  typeParameterStart = 29184, // typeParameterStart
+  nonNullAssertion = 29696, // nonNullAssertion
+  _break = 30224, // break keyword
+  _case = 30736, // case keyword
+  _catch = 31248, // catch keyword
+  _continue = 31760, // continue keyword
+  _debugger = 32272, // debugger keyword
+  _default = 32784, // default keyword
+  _do = 33296, // do keyword
+  _else = 33808, // else keyword
+  _finally = 34320, // finally keyword
+  _for = 34832, // for keyword
+  _function = 35344, // function keyword
+  _if = 35856, // if keyword
+  _return = 36368, // return keyword
+  _switch = 36880, // switch keyword
+  _throw = 37520, // throw keyword prefix
+  _try = 37904, // try keyword
+  _var = 38416, // var keyword
+  _let = 38928, // let keyword
+  _const = 39440, // const keyword
+  _while = 39952, // while keyword
+  _with = 40464, // with keyword
+  _new = 40976, // new keyword
+  _this = 41488, // this keyword
+  _super = 42000, // super keyword
+  _class = 42512, // class keyword
+  _extends = 43024, // extends keyword
+  _export = 43536, // export keyword
+  _import = 44048, // import keyword
+  _yield = 44560, // yield keyword
+  _null = 45072, // null keyword
+  _true = 45584, // true keyword
+  _false = 46096, // false keyword
+  _in = 46616, // in prec:8 keyword
+  _instanceof = 47128, // instanceof prec:8 keyword
+  _typeof = 47760, // typeof keyword prefix
+  _void = 48272, // void keyword prefix
+  _delete = 48784, // delete keyword prefix
+  _async = 49168, // async keyword
+  _get = 49680, // get keyword
+  _set = 50192, // set keyword
+  _declare = 50704, // declare keyword
+  _readonly = 51216, // readonly keyword
+  _abstract = 51728, // abstract keyword
+  _static = 52240, // static keyword
+  _public = 52752, // public keyword
+  _private = 53264, // private keyword
+  _protected = 53776, // protected keyword
+  _as = 54288, // as keyword
+  _enum = 54800, // enum keyword
+  _type = 55312, // type keyword
+  _implements = 55824, // implements keyword
 }
 export function formatTokenType(tokenType: TokenType): string {
   switch (tokenType) {
     case TokenType.num:
       return "num";
     case TokenType.bigint:
       return "bigint";
+    case TokenType.decimal:
+      return "decimal";
     case TokenType.regexp:
       return "regexp";
     case TokenType.string: