Port babel-parser changes from 2023-03-20 to 2023-06-15

Instructions: https://github.com/alangpierce/sucrase/wiki/Porting-changes-from-Babel's-parser

032203ea18 chore: Update TS 5.0.2 (#15502)
🚫 Babel-internal change.

656403b49f refactor: introduce `lookaheadInLineCharCode` (#15510)
🚫 I considered porting this refactor for `await using`, but decided against it.

202e4fa60b v7.21.4
🚫 Release only.

35c09db4f3 perf: Improve the code quality of `@babe/parser` for ts (#15540)
🚫 Code changes are minor and generally don't affect Sucrase.

bc7f795024 fix: Remove `mixins` and `implements` for `DeclareInterface` and `InterfaceDeclaration` (#15539)
🚫 AST-only.

0a1cfc0ed6 Babel 8 misc changes (#15068)
🚫 Comment-only change not affecting Sucrase.

86df74531d v7.21.5
🚫 Release only.

29426297d6 Remove `using await` restriction in explicitResourceManagement (#15602)
🚫 Only affects error handling, not relevant to Sucrase.

8fd99c8f48 v7.21.8
🚫 Release only.

2b70257372 chore: Enable rule `no-import-type-side-effects` (#15627)
🚫 Babel-internal change.

c5f68eb62e rescan gt token at the end of type args parsing (#15631)
✅ Added test and implemented fix similarly. Fixes #797.

0ad567f913 v7.21.9
🚫 Release only.

920b78c0c2 Enable regexp unicode sets parsing by default (#15638)
🚫 Doesn't affect Sucrase.

56a6c85aca Parse `await using` declarations (#15520)
✅ Implemented using a simpler backtracking method for now.

4b5ebdc9b3 Add support for the updated import attributes proposal (#15536)
✅ Implemented with a small tweak to Sucrase implementation.

7e5e7bf2f8 Unify parsing of import/export modifiers (type/typeof/module) (#15630)
🚫 Skipped for now, since it's prep work for later import syntax changes and I want to wait for that to settle a little more.

560b8ac523 [ts] Support `import ... =` and `export =` in scripts (#15497)
🚫 Sucrase already supported this behavior.

389ecb08ed v7.22.0
🚫 Release only.

325fe683d8 v7.22.3
🚫 Release only.

35116224d2 Mark `assert` attributes with `extra.deprecatedAssertSyntax` (#15667)
🚫 AST only.

eb4aa876b2 v7.22.4
🚫 Release only.

ecc819bf75 [babel 8] Require Node.js `^16.20.0 || ^18.16.0 || >=20.0.0` (#15585)
🚫 Babel-internal change.

72006cb657 Use Prettier v3.0.0-alpha.12 (#15617)
🚫 Babel-internal change.

08564ea230 v7.22.5
🚫 Release only.

be8fccd6f3 chore: Run `readmes.js` in CI (#15690)
🚫 Babel-internal change.

980b63c829 Use more optional chaining (#15703)
🚫 Babel-internal.

src/parser/plugins/typescript.ts

@@ -9,6 +9,7 @@ import {
   nextTemplateToken,
   popTypeContext,
   pushTypeContext,
+  rescan_gt,
 } from "../tokenizer/index";
 import {ContextualKeyword} from "../tokenizer/keywords";
 import {TokenType, TokenType as tt} from "../tokenizer/types";
@@ -1153,11 +1154,26 @@ function tsParseTypeArgumentsWithPossibleBitshift(): void {
 function tsParseTypeArguments(): void {
   const oldIsType = pushTypeContext(0);
   expect(tt.lessThan);
-  while (!eat(tt.greaterThan) && !state.error) {
+  while (!match(tt.greaterThan) && !state.error) {
     tsParseType();
     eat(tt.comma);
   }
-  popTypeContext(oldIsType);
+  if (!oldIsType) {
+    // If the type arguments are present in an expression context, e.g.
+    // f<number>(), then the > sign should be tokenized as a non-type token.
+    // In particular, f(a < b, c >= d) should parse the >= as a single token,
+    // resulting in a syntax error and fallback to the non-type-args
+    // interpretation. In the success case, even though the > is tokenized as a
+    // non-type token, it still must be marked as a type token so that it is
+    // erased.
+    popTypeContext(oldIsType);
+    rescan_gt();
+    expect(tt.greaterThan);
+    state.tokens[state.tokens.length - 1].isType = true;
+  } else {
+    expect(tt.greaterThan);
+    popTypeContext(oldIsType);
+  }
 }
 
 export function tsIsDeclarationStart(): boolean {

src/parser/tokenizer/index.ts

@@ -571,15 +571,18 @@ function readToken_gt(): void {
 }
 
 /**
- * Called after `as` expressions in TS; we're switching from a type to a
- * non-type context, so a > token may actually be >= . This is needed because >=
- * must be tokenized as a > in a type context because of code like
- * `const x: Array<T>=[];`, but `a as T >= 1` is a code example where it must be
- * treated as >=.
+ * Reinterpret a possible > token when transitioning from a type to a non-type
+ * context.
  *
- * Notably, this only applies to >, not <. In a code snippet like `a as T <= 1`,
- * we must NOT tokenize as <, or else the type parser will start parsing a type
- * argument and fail.
+ * This comes up in two situations where >= needs to be treated as one token:
+ * - After an `as` expression, like in the code `a as T >= 1`.
+ * - In a type argument in an expression context, e.g. `f(a < b, c >= d)`, we
+ *   need to see the token as >= so that we get an error and backtrack to
+ *   normal expression parsing.
+ *
+ * Other situations require >= to be seen as two tokens, e.g.
+ * `const x: Array<T>=[];`, so it's important to treat > as its own token in
+ * typical type parsing situations.
  */
 export function rescan_gt(): void {
   if (state.type === tt.greaterThan) {

src/parser/traverser/statement.ts

@@ -223,6 +223,10 @@ function parseStatementContent(declaration: boolean): void {
       ) {
         parseVarStatement(true);
         return;
+      } else if (startsAwaitUsing()) {
+        expectContextual(ContextualKeyword._await);
+        parseVarStatement(true);
+        return;
       }
     default:
       // Do nothing.
@@ -255,6 +259,47 @@ function parseStatementContent(declaration: boolean): void {
   }
 }
 
+/**
+ * Determine if we're positioned at an `await using` declaration.
+ *
+ * Note that this can happen either in place of a regular variable declaration
+ * or in a loop body, and in both places, there are
+ *
+ * Examples returning true:
+ * await using foo = bar();
+ * for (await using a of b) {}
+ *
+ * Examples returning false:
+ * await using
+ * await using + 1
+ * await using instanceof T
+ * for (await using;;) {}
+ *
+ * For now, we do a simple backtracking-based lookahead for the `using` and
+ * identifier tokens. In the future, this could be optimized with a
+ * character-based approach.
+ */
+function startsAwaitUsing(): boolean {
+  if (!isContextual(ContextualKeyword._await)) {
+    return false;
+  }
+  const snapshot = state.snapshot();
+  // await
+  next();
+  if (!isContextual(ContextualKeyword._using) || hasPrecedingLineBreak()) {
+    state.restoreFromSnapshot(snapshot);
+    return false;
+  }
+  // using
+  next();
+  if (!match(tt.name) || hasPrecedingLineBreak()) {
+    state.restoreFromSnapshot(snapshot);
+    return false;
+  }
+  state.restoreFromSnapshot(snapshot);
+  return true;
+}
+
 export function parseDecorators(): void {
   while (match(tt.at)) {
     parseDecorator();
@@ -361,7 +406,11 @@ function parseAmbiguousForStatement(): void {
     return;
   }
 
-  if (match(tt._var) || match(tt._let) || match(tt._const) || isUsingInLoop()) {
+  const isAwaitUsing = startsAwaitUsing();
+  if (isAwaitUsing || match(tt._var) || match(tt._let) || match(tt._const) || isUsingInLoop()) {
+    if (isAwaitUsing) {
+      expectContextual(ContextualKeyword._await);
+    }
     next();
     parseVar(true, state.type !== tt._var);
     if (match(tt._in) || isContextual(ContextualKeyword._of)) {
@@ -1024,7 +1073,7 @@ function parseExportSpecifiersMaybe(): void {
 export function parseExportFrom(): void {
   if (eatContextual(ContextualKeyword._from)) {
     parseExprAtom();
-    maybeParseImportAssertions();
+    maybeParseImportAttributes();
   }
   semicolon();
 }
@@ -1192,7 +1241,7 @@ export function parseImport(): void {
     expectContextual(ContextualKeyword._from);
     parseExprAtom();
   }
-  maybeParseImportAssertions();
+  maybeParseImportAttributes();
   semicolon();
 }
 
@@ -1268,13 +1317,14 @@ function parseImportSpecifier(): void {
 }
 
 /**
- * Parse import assertions like `assert {type: "json"}`.
+ * Parse import attributes like `with {type: "json"}`, or the legacy form
+ * `assert {type: "json"}`.
  *
- * Import assertions technically have their own syntax, but are always parseable
+ * Import attributes technically have their own syntax, but are always parseable
  * as a plain JS object, so just do that for simplicity.
  */
-function maybeParseImportAssertions(): void {
-  if (isContextual(ContextualKeyword._assert) && !hasPrecedingLineBreak()) {
+function maybeParseImportAttributes(): void {
+  if (match(tt._with) || (isContextual(ContextualKeyword._assert) && !hasPrecedingLineBreak())) {
     next();
     parseObj(false, false);
   }

src/transformers/CJSImportTransformer.ts

@@ -11,7 +11,7 @@ import getDeclarationInfo, {
   EMPTY_DECLARATION_INFO,
 } from "../util/getDeclarationInfo";
 import getImportExportSpecifierInfo from "../util/getImportExportSpecifierInfo";
-import {removeMaybeImportAssertion} from "../util/removeMaybeImportAssertion";
+import {removeMaybeImportAttributes} from "../util/removeMaybeImportAttributes";
 import shouldElideDefaultExport from "../util/shouldElideDefaultExport";
 import type ReactHotLoaderTransformer from "./ReactHotLoaderTransformer";
 import type RootTransformer from "./RootTransformer";
@@ -150,7 +150,7 @@ export default class CJSImportTransformer extends Transformer {
       this.tokens.replaceTokenTrimmingLeftWhitespace(this.importProcessor.claimImportCode(path));
       this.tokens.appendCode(this.importProcessor.claimImportCode(path));
     }
-    removeMaybeImportAssertion(this.tokens);
+    removeMaybeImportAttributes(this.tokens);
     if (this.tokens.matches1(tt.semi)) {
       this.tokens.removeToken();
     }
@@ -354,7 +354,7 @@ export default class CJSImportTransformer extends Transformer {
       ) {
         this.tokens.removeToken();
         this.tokens.removeToken();
-        removeMaybeImportAssertion(this.tokens);
+        removeMaybeImportAttributes(this.tokens);
       }
       return true;
     } else {
@@ -829,7 +829,7 @@ export default class CJSImportTransformer extends Transformer {
       this.tokens.removeToken();
       const path = this.tokens.stringValue();
       this.tokens.replaceTokenTrimmingLeftWhitespace(this.importProcessor.claimImportCode(path));
-      removeMaybeImportAssertion(this.tokens);
+      removeMaybeImportAttributes(this.tokens);
     } else {
       // This is a normal named export, so use that.
       this.tokens.appendCode(exportStatements.join(" "));
@@ -847,7 +847,7 @@ export default class CJSImportTransformer extends Transformer {
     }
     const path = this.tokens.stringValue();
     this.tokens.replaceTokenTrimmingLeftWhitespace(this.importProcessor.claimImportCode(path));
-    removeMaybeImportAssertion(this.tokens);
+    removeMaybeImportAttributes(this.tokens);
     if (this.tokens.matches1(tt.semi)) {
       this.tokens.removeToken();
     }

src/transformers/ESMImportTransformer.ts

@@ -11,7 +11,7 @@ import getDeclarationInfo, {
 } from "../util/getDeclarationInfo";
 import getImportExportSpecifierInfo from "../util/getImportExportSpecifierInfo";
 import {getNonTypeIdentifiers} from "../util/getNonTypeIdentifiers";
-import {removeMaybeImportAssertion} from "../util/removeMaybeImportAssertion";
+import {removeMaybeImportAttributes} from "../util/removeMaybeImportAttributes";
 import shouldElideDefaultExport from "../util/shouldElideDefaultExport";
 import type ReactHotLoaderTransformer from "./ReactHotLoaderTransformer";
 import Transformer from "./Transformer";
@@ -118,7 +118,7 @@ export default class ESMImportTransformer extends Transformer {
       ) {
         this.tokens.removeToken();
         this.tokens.removeToken();
-        removeMaybeImportAssertion(this.tokens);
+        removeMaybeImportAttributes(this.tokens);
       }
       return true;
     }
@@ -162,7 +162,7 @@ export default class ESMImportTransformer extends Transformer {
         this.tokens.removeToken();
       }
       this.tokens.removeToken();
-      removeMaybeImportAssertion(this.tokens);
+      removeMaybeImportAttributes(this.tokens);
       if (this.tokens.matches1(tt.semi)) {
         this.tokens.removeToken();
       }

src/util/removeMaybeImportAttributes.ts

@@ -0,0 +1,22 @@
+import {ContextualKeyword} from "../parser/tokenizer/keywords";
+import {TokenType as tt} from "../parser/tokenizer/types";
+import type TokenProcessor from "../TokenProcessor";
+
+/**
+ * Starting at a potential `with` or (legacy) `assert` token, remove the import
+ * attributes if they exist.
+ */
+export function removeMaybeImportAttributes(tokens: TokenProcessor): void {
+  if (
+    tokens.matches2(tt._with, tt.braceL) ||
+    (tokens.matches2(tt.name, tt.braceL) && tokens.matchesContextual(ContextualKeyword._assert))
+  ) {
+    // assert
+    tokens.removeToken();
+    // {
+    tokens.removeToken();
+    tokens.removeBalancedCode();
+    // }
+    tokens.removeToken();
+  }
+}

test/imports-test.ts

@@ -327,7 +327,7 @@ return obj && obj.__esModule ? obj : { default: obj }; }
     );
   });
 
-  it("removes import assertions", () => {
+  it("removes legacy import assertions", () => {
     assertResult(
       `
       import DefaultName from 'module1' assert {type: "json"};
@@ -350,6 +350,29 @@ return obj && obj.__esModule ? obj : { default: obj }; }
     );
   });
 
+  it("removes import attributes", () => {
+    assertResult(
+      `
+      import DefaultName from 'module1' with {type: "json"};
+      import {namedName} from 'module2' with {type: "json"};
+      import "module3" with {type: "json"};
+      export * from "module4" with {type: "json"};
+      // Arbitrary expressions like these aren't actually allowed right now, but
+      // exercise the ability to detect matching braces.
+      import test from "module5" with {type: {foo: "test"}};
+    `,
+      `"use strict";${ESMODULE_PREFIX}${IMPORT_DEFAULT_PREFIX}${CREATE_STAR_EXPORT_PREFIX}
+      var _module1 = require('module1'); var _module12 = _interopRequireDefault(_module1);
+      var _module2 = require('module2');
+      require('module3');
+      var _module4 = require('module4'); _createStarExport(_module4);
+      // Arbitrary expressions like these aren't actually allowed right now, but
+      // exercise the ability to detect matching braces.
+      var _module5 = require('module5'); var _module52 = _interopRequireDefault(_module5);
+    `,
+    );
+  });
+
   it("allows an import statement with no import bindings", () => {
     assertResult(
       `

test/sucrase-test.ts

@@ -1780,4 +1780,53 @@ describe("sucrase", () => {
       {transforms: []},
     );
   });
+
+  it("correctly handles `await using`", () => {
+    assertResult(
+      `
+      async function foo() {
+        await using x = blah();
+      }
+    `,
+      `
+      async function foo() {
+        await using x = blah();
+      }
+    `,
+      {transforms: []},
+    );
+  });
+  it("correctly handles `await using` in a loop", () => {
+    assertResult(
+      `
+      async function foo() {
+        for (await using a of b) {}
+      }
+    `,
+      `
+      async function foo() {
+        for (await using a of b) {}
+      }
+    `,
+      {transforms: []},
+    );
+  });
+
+  it("is not confused `await using` in other contexts", () => {
+    assertResult(
+      `
+      await using
+      await using instanceof Foo
+      await using + 1
+      for (await using;;) {}
+    `,
+      `
+      await using
+      await using instanceof Foo
+      await using + 1
+      for (await using;;) {}
+    `,
+      {transforms: []},
+    );
+  });
 });