[babel 8] Improve syntax highlighting (#12660)

Co-authored-by: Nicolò Ribaudo <nicolo.ribaudo@gmail.com>
Co-authored-by: Simon Lydell <simon.lydell@gmail.com>

packages/babel-code-frame/test/index.js

@@ -283,6 +283,36 @@ describe("@babel/code-frame", function () {
     );
   });
 
+  test("jsx", function () {
+    const gutter = chalk.grey;
+    const yellow = chalk.yellow;
+
+    const rawLines = ["<div />"].join("\n");
+
+    expect(
+      JSON.stringify(
+        codeFrame(rawLines, 0, null, {
+          linesAbove: 1,
+          linesBelow: 1,
+          forceColor: true,
+        }),
+      ),
+    ).toEqual(
+      JSON.stringify(
+        chalk.reset(
+          " " +
+            gutter(" 1 |") +
+            " " +
+            yellow("<") +
+            yellow("div") +
+            " " +
+            yellow("/") +
+            yellow(">"),
+        ),
+      ),
+    );
+  });
+
   test("basic usage, new API", function () {
     const rawLines = ["class Foo {", "  constructor()", "};"].join("\n");
     expect(

packages/babel-highlight/package.json

@@ -17,7 +17,7 @@
   "dependencies": {
     "@babel/helper-validator-identifier": "workspace:^7.10.4",
     "chalk": "^2.0.0",
-    "js-tokens": "^4.0.0"
+    "js-tokens": "condition:BABEL_8_BREAKING ? ^6.0.0 : ^4.0.0"
   },
   "devDependencies": {
     "strip-ansi": "^4.0.0"

packages/babel-highlight/src/index.js

@@ -1,17 +1,30 @@
-import jsTokens, { matchToToken } from "js-tokens";
-import { isReservedWord, isKeyword } from "@babel/helper-validator-identifier";
+import jsTokens, * as jsTokensNs from "js-tokens";
+import {
+  isStrictReservedWord,
+  isKeyword,
+} from "@babel/helper-validator-identifier";
 import Chalk from "chalk";
 
+/**
+ * Names that are always allowed as identifiers, but also appear as keywords
+ * within certain syntactic productions.
+ *
+ * https://tc39.es/ecma262/#sec-keywords-and-reserved-words
+ *
+ * `target` has been omitted since it is very likely going to be a false
+ * positive.
+ */
+const sometimesKeywords = new Set(["as", "async", "from", "get", "of", "set"]);
+
 /**
  * Chalk styles for token types.
  */
 function getDefs(chalk) {
   return {
     keyword: chalk.cyan,
     capitalized: chalk.yellow,
-    jsx_tag: chalk.yellow,
+    jsxIdentifier: chalk.yellow,
     punctuator: chalk.yellow,
-    // bracket:  intentionally omitted.
     number: chalk.magenta,
     string: chalk.green,
     regex: chalk.magenta,
@@ -25,72 +38,188 @@ function getDefs(chalk) {
  */
 const NEWLINE = /\r\n|[\n\r\u2028\u2029]/;
 
-/**
- * RegExp to test for what seems to be a JSX tag name.
- */
-const JSX_TAG = /^[a-z][\w-]*$/i;
-
 /**
  * RegExp to test for the three types of brackets.
  */
 const BRACKET = /^[()[\]{}]$/;
 
-/**
- * Get the type of token, specifying punctuator type.
- */
-function getTokenType(match) {
-  const [offset, text] = match.slice(-2);
-  const token = matchToToken(match);
+let tokenize;
+
+if (process.env.BABEL_8_BREAKING) {
+  /**
+   * Get the type of token, specifying punctuator type.
+   */
+  const getTokenType = function (token) {
+    if (token.type === "IdentifierName") {
+      if (
+        isKeyword(token.value) ||
+        isStrictReservedWord(token.value, true) ||
+        sometimesKeywords.has(token.value)
+      ) {
+        return "keyword";
+      }
+
+      if (token.value[0] !== token.value[0].toLowerCase()) {
+        return "capitalized";
+      }
+    }
 
-  if (token.type === "name") {
-    if (isKeyword(token.value) || isReservedWord(token.value)) {
-      return "keyword";
+    if (token.type === "Punctuator" && BRACKET.test(token.value)) {
+      return "uncolored";
     }
 
     if (
-      JSX_TAG.test(token.value) &&
-      (text[offset - 1] === "<" || text.substr(offset - 2, 2) == "</")
+      token.type === "Invalid" &&
+      (token.value === "@" || token.value === "#")
     ) {
-      return "jsx_tag";
+      return "punctuator";
     }
 
-    if (token.value[0] !== token.value[0].toLowerCase()) {
-      return "capitalized";
+    switch (token.type) {
+      case "NumericLiteral":
+        return "number";
+
+      case "StringLiteral":
+      case "JSXString":
+      case "NoSubstitutionTemplate":
+        return "string";
+
+      case "RegularExpressionLiteral":
+        return "regex";
+
+      case "Punctuator":
+      case "JSXPunctuator":
+        return "punctuator";
+
+      case "MultiLineComment":
+      case "SingleLineComment":
+        return "comment";
+
+      case "Invalid":
+      case "JSXInvalid":
+        return "invalid";
+
+      case "JSXIdentifier":
+        return "jsxIdentifier";
+
+      default:
+        return "uncolored";
     }
-  }
+  };
 
-  if (token.type === "punctuator" && BRACKET.test(token.value)) {
-    return "bracket";
-  }
+  /**
+   * Turn a string of JS into an array of objects.
+   */
+  tokenize = function* (text: string) {
+    for (const token of jsTokens(text, { jsx: true })) {
+      switch (token.type) {
+        case "TemplateHead":
+          yield { type: "string", value: token.value.slice(0, -2) };
+          yield { type: "punctuator", value: "${" };
+          break;
 
-  if (
-    token.type === "invalid" &&
-    (token.value === "@" || token.value === "#")
-  ) {
-    return "punctuator";
-  }
+        case "TemplateMiddle":
+          yield { type: "punctuator", value: "}" };
+          yield { type: "string", value: token.value.slice(1, -2) };
+          yield { type: "punctuator", value: "${" };
+          break;
 
-  return token.type;
+        case "TemplateTail":
+          yield { type: "punctuator", value: "}" };
+          yield { type: "string", value: token.value.slice(1) };
+          break;
+
+        default:
+          yield {
+            type: getTokenType(token),
+            value: token.value,
+          };
+      }
+    }
+  };
+} else {
+  // This is only available in js-tokens@4, and not in js-tokens@6
+  const { matchToToken } = jsTokensNs;
+
+  /**
+   * RegExp to test for what seems to be a JSX tag name.
+   */
+  const JSX_TAG = /^[a-z][\w-]*$/i;
+
+  const getTokenType = function (token, offset, text) {
+    if (token.type === "name") {
+      if (
+        isKeyword(token.value) ||
+        isStrictReservedWord(token.value, true) ||
+        sometimesKeywords.has(token.value)
+      ) {
+        return "keyword";
+      }
+
+      if (
+        JSX_TAG.test(token.value) &&
+        (text[offset - 1] === "<" || text.substr(offset - 2, 2) == "</")
+      ) {
+        return "jsxIdentifier";
+      }
+
+      if (token.value[0] !== token.value[0].toLowerCase()) {
+        return "capitalized";
+      }
+    }
+
+    if (token.type === "punctuator" && BRACKET.test(token.value)) {
+      return "bracket";
+    }
+
+    if (
+      token.type === "invalid" &&
+      (token.value === "@" || token.value === "#")
+    ) {
+      return "punctuator";
+    }
+
+    return token.type;
+  };
+
+  tokenize = function* (text: string) {
+    let match;
+    while ((match = jsTokens.exec(text))) {
+      const token = matchToToken(match);
+
+      yield {
+        type: getTokenType(token, match.index, text),
+        value: token.value,
+      };
+    }
+  };
 }
 
 /**
  * Highlight `text` using the token definitions in `defs`.
  */
 function highlightTokens(defs: Object, text: string) {
-  return text.replace(jsTokens, function (...args) {
-    const type = getTokenType(args);
+  let highlighted = "";
+
+  for (const { type, value } of tokenize(text)) {
     const colorize = defs[type];
     if (colorize) {
-      return args[0]
+      highlighted += value
         .split(NEWLINE)
         .map(str => colorize(str))
         .join("\n");
     } else {
-      return args[0];
+      highlighted += value;
     }
-  });
+  }
+
+  return highlighted;
 }
 
+/**
+ * Highlight `text` using the token definitions in `defs`.
+ */
+
 type Options = {
   forceColor?: boolean,
 };

yarn.lock

@@ -858,7 +858,7 @@ __metadata:
   dependencies:
     "@babel/helper-validator-identifier": "workspace:^7.10.4"
     chalk: ^2.0.0
-    js-tokens: ^4.0.0
+    js-tokens: "condition:BABEL_8_BREAKING ? ^6.0.0 : ^4.0.0"
     strip-ansi: ^4.0.0
   languageName: unknown
   linkType: soft
@@ -9060,13 +9060,30 @@ fsevents@^1.2.7:
   languageName: node
   linkType: hard
 
-"js-tokens@npm:^4.0.0":
+"js-tokens-BABEL_8_BREAKING-false@npm:js-tokens@^4.0.0, js-tokens@npm:^4.0.0":
   version: 4.0.0
   resolution: "js-tokens@npm:4.0.0"
   checksum: 1fc4e4667ac2d972aba65148b9cbf9c17566b2394d3504238d8492bbd3e68f496c657eab06b26b40b17db5cac0a34d153a12130e2d2d2bb6dc2cdc8a4764eb1b
   languageName: node
   linkType: hard
 
+"js-tokens-BABEL_8_BREAKING-true@npm:js-tokens@^6.0.0":
+  version: 6.0.0
+  resolution: "js-tokens@npm:6.0.0"
+  checksum: 975859a4fd68cbaaabf106639df316e662b87b296afa9c6b00cfd25bc7642137433d18bf78e1e5578fc63c2a3e7334aad4fbed47f87c6c29f9a4f6760e79e322
+  languageName: node
+  linkType: hard
+
+"js-tokens@condition:BABEL_8_BREAKING ? ^6.0.0 : ^4.0.0":
+  version: 0.0.0-condition-bceac3
+  resolution: "js-tokens@condition:BABEL_8_BREAKING?^6.0.0:^4.0.0#bceac3"
+  dependencies:
+    js-tokens-BABEL_8_BREAKING-false: "npm:js-tokens@^4.0.0"
+    js-tokens-BABEL_8_BREAKING-true: "npm:js-tokens@^6.0.0"
+  checksum: 036166b3ba76e31549eeb404d986ff5b1af55f91137bbcc6d5147b1e4c8d4c74f01d9aae10cf5d5221e60f3bcef98e7460bbf2a54a9e7b47d3b63789b11297e3
+  languageName: node
+  linkType: hard
+
 "js-yaml@npm:^3.13.1, js-yaml@npm:^3.2.1":
   version: 3.13.1
   resolution: "js-yaml@npm:3.13.1"