Port babel-parser changes from 2022-01-10 to 2022-02-26 (alangpierce#716

)

Instructions: https://github.com/alangpierce/sucrase/wiki/Porting-changes-from-Babel's-parser

69246b6212 [babel 8] fix properties name for function-like TS nodes (#13709)
🚫 AST only.

bc0d1ef860 v7.16.8
🚫 Release only.

478a9709ab Improve errors location tracking (#14130)
🚫 Change to error messaging details that were removed in Sucrase.

4a737547e5 v7.16.10
🚫 Release only.

e5d29f6e40 fix: incorrect conciseBody lookahead (#14194)
✅ Bug doesn't appear in Sucrase, but I added a test anyway.

5266605528 v7.16.12
🚫 Release only.

5861002b33 Reinterpret << when parsing TS type arguments (#14145)
✅ Ported following a very similar implementation.

6b427ced22 Fuzz test location-related parser options (#14201)
🚫 Babel-internal change testing location logic not present in Sucrase.

96a8251def Add parser support for the "regexp unicode sets" proposal (#14086)
✅ No implementation necessary, but I added a quick test.

f893b333a8 Add the `decoratorsAutoAccessors` parser plugin (#13681)
✅ Ported basic parsing for accessor properties.

de5c7b1bce Parse destructuring private fields (#13931)
✅ Seems to be already working, and I added a test.

🚫 d50c18dbc2 fix: usePrivateName usage
Only affects Babel scope code, not relevant to Sucrase.

🚫 b092bd0cb5 remove invalid test output
Only affects Babel tests.

df27d542ef proposal-pipe: Add support for `^^` and `@@` topics (#13973)
🚫 Sucrase is holding off on pipeline topic parsing until the details are finalized. Tracked in alangpierce#674.

97a8bcb9cc Expose `.index` on Position to internally track nodes location (#14174)
🚫 Only affects class not used in Sucrase.

38c23cded4 v7.17.0
🚫 Release only.

19ede090eb parser: Update `babel-parser.d.ts` for 7v.17.0 features (#14266)
🚫 Not relevant to Sucrase.

f52c70c357 v7.17.3
🚫 Release only.

a53c2fa4a2 fix(ts): skip func-type param start on parsing (#14293)
🚫 Bug not present in Sucrase.

5749c16dc2 [tsx] raise error on single arrow type argument without comma (#14135)
🚫 Only affects error handling.

generator/generateReadWordTree.ts

@@ -42,6 +42,7 @@ const KEYWORDS = [
 
 const CONTEXTUAL_KEYWORDS = [
   "abstract",
+  "accessor",
   "as",
   "asserts",
   "async",

generator/generateTokenTypes.ts

@@ -97,7 +97,8 @@ const types = {
   lessThan: new BinopTokenType("<", 7),
   greaterThan: new BinopTokenType(">", 7),
   relationalOrEqual: new BinopTokenType("<=/>=", 7),
-  bitShift: new BinopTokenType("<</>>", 8),
+  bitShiftL: new BinopTokenType("<<", 8),
+  bitShiftR: new BinopTokenType(">>/>>>", 8),
   plus: new TokenType("+", {binop: 9, prefix}),
   minus: new TokenType("-", {binop: 9, prefix}),
   modulo: new BinopTokenType("%", 10),
@@ -173,6 +174,9 @@ const types = {
 
 export default function generateTokenTypes(): string {
   let code = '// Generated file, do not edit! Run "yarn generate" to re-generate this file.\n';
+  // formatTokenType is trivial and used for debugging purposes, so we shouldn't
+  // need full test coverage.
+  code += "/* istanbul ignore file */\n";
   code += generateTokenTypeEnum();
   code += generateFormatTokenType();
   return code;

src/parser/plugins/typescript.ts

@@ -1,5 +1,6 @@
 import {
   eat,
+  finishToken,
   IdentifierRole,
   lookaheadType,
   lookaheadTypeAndKeyword,
@@ -1095,6 +1096,25 @@ function tsTryParseGenericAsyncArrowFunction(): boolean {
   return true;
 }
 
+/**
+ * If necessary, hack the tokenizer state so that this bitshift was actually a
+ * less-than token, then keep parsing. This should only be used in situations
+ * where we restore from snapshot on error (which reverts this change) or
+ * where bitshift would be illegal anyway (e.g. in a class "extends" clause).
+ *
+ * This hack is useful to handle situations like foo<<T>() => void>() where
+ * there can legitimately be two open-angle-brackets in a row in TS. This
+ * situation is very obscure and (as of this writing) is handled by Babel but
+ * not TypeScript itself, so it may be fine in the future to remove this case.
+ */
+function tsParseTypeArgumentsWithPossibleBitshift(): void {
+  if (state.type === tt.bitShiftL) {
+    state.pos -= 1;
+    finishToken(tt.lessThan);
+  }
+  tsParseTypeArguments();
+}
+
 function tsParseTypeArguments(): void {
   const oldIsType = pushTypeContext(0);
   expect(tt.lessThan);
@@ -1165,7 +1185,7 @@ export function tsParseSubscript(
     return;
   }
 
-  if (match(tt.lessThan)) {
+  if (match(tt.lessThan) || match(tt.bitShiftL)) {
     // There are number of things we are going to "maybe" parse, like type arguments on
     // tagged template expressions. If any of them fail, walk it back and continue.
     const snapshot = state.snapshot();
@@ -1178,7 +1198,7 @@ export function tsParseSubscript(
         return;
       }
     }
-    tsParseTypeArguments();
+    tsParseTypeArgumentsWithPossibleBitshift();
     if (!noCalls && eat(tt.parenL)) {
       // With f<T>(), the subscriptStartIndex marker is on the ( token.
       state.tokens[state.tokens.length - 1].subscriptStartIndex = startTokenIndex;
@@ -1210,15 +1230,11 @@ export function tsParseSubscript(
 }
 
 export function tsStartParseNewArguments(): void {
-  if (match(tt.lessThan)) {
+  if (match(tt.lessThan) || match(tt.bitShiftL)) {
     // 99% certain this is `new C<T>();`. But may be `new C < T;`, which is also legal.
     const snapshot = state.snapshot();
 
-    state.type = tt.typeParameterStart;
-    tsParseTypeArguments();
-    if (!match(tt.parenL)) {
-      unexpected();
-    }
+    tsParseTypeArgumentsWithPossibleBitshift();
 
     if (state.error) {
       state.restoreFromSnapshot(snapshot);
@@ -1431,8 +1447,8 @@ export function tsParseExportDeclaration(): void {
 }
 
 export function tsAfterParseClassSuper(hasSuper: boolean): void {
-  if (hasSuper && match(tt.lessThan)) {
-    tsParseTypeArguments();
+  if (hasSuper && (match(tt.lessThan) || match(tt.bitShiftL))) {
+    tsParseTypeArgumentsWithPossibleBitshift();
   }
   if (eatContextual(ContextualKeyword._implements)) {
     state.tokens[state.tokens.length - 1].type = tt._implements;
@@ -1553,8 +1569,8 @@ export function tsParseAssignableListItemTypes(): void {
 }
 
 export function tsParseMaybeDecoratorArguments(): void {
-  if (match(tt.lessThan)) {
-    tsParseTypeArguments();
+  if (match(tt.lessThan) || match(tt.bitShiftL)) {
+    tsParseTypeArgumentsWithPossibleBitshift();
   }
   baseParseMaybeDecoratorArguments();
 }

src/parser/tokenizer/index.ts

@@ -482,15 +482,36 @@ function readToken_plus_min(code: number): void {
   }
 }
 
-// '<>'
-function readToken_lt_gt(code: number): void {
+function readToken_lt(): void {
+  const nextChar = input.charCodeAt(state.pos + 1);
+
+  if (nextChar === charCodes.lessThan) {
+    if (input.charCodeAt(state.pos + 2) === charCodes.equalsTo) {
+      finishOp(tt.assign, 3);
+      return;
+    }
+    // This still might be two instances of <, e.g. the TS type argument
+    // expression f<<T>() => void>() , but parse as left shift for now and we'll
+    // retokenize if necessary. We can't use isType for this case because we
+    // don't know yet if we're in a type.
+    finishOp(tt.bitShiftL, 2);
+    return;
+  }
+
+  if (nextChar === charCodes.equalsTo) {
+    // <=
+    finishOp(tt.relationalOrEqual, 2);
+  } else {
+    finishOp(tt.lessThan, 1);
+  }
+}
+
+function readToken_gt(): void {
+  const code = charCodes.greaterThan;
   const nextChar = input.charCodeAt(state.pos + 1);
 
   if (nextChar === code) {
-    const size =
-      code === charCodes.greaterThan && input.charCodeAt(state.pos + 2) === charCodes.greaterThan
-        ? 3
-        : 2;
+    const size = input.charCodeAt(state.pos + 2) === charCodes.greaterThan ? 3 : 2;
     if (input.charCodeAt(state.pos + size) === charCodes.equalsTo) {
       finishOp(tt.assign, size + 1);
       return;
@@ -500,15 +521,13 @@ function readToken_lt_gt(code: number): void {
       finishOp(tt.greaterThan, 1);
       return;
     }
-    finishOp(tt.bitShift, size);
+    finishOp(tt.bitShiftR, size);
     return;
   }
 
   if (nextChar === charCodes.equalsTo) {
-    // <= | >=
+    // >=
     finishOp(tt.relationalOrEqual, 2);
-  } else if (code === charCodes.lessThan) {
-    finishOp(tt.lessThan, 1);
   } else {
     finishOp(tt.greaterThan, 1);
   }
@@ -695,8 +714,11 @@ export function getTokenFromCode(code: number): void {
       return;
 
     case charCodes.lessThan:
+      readToken_lt();
+      return;
+
     case charCodes.greaterThan:
-      readToken_lt_gt(code);
+      readToken_gt();
       return;
 
     case charCodes.equalsTo:

src/parser/tokenizer/keywords.ts

@@ -1,6 +1,7 @@
 export enum ContextualKeyword {
   NONE,
   _abstract,
+  _accessor,
   _as,
   _asserts,
   _async,