Improve syntax highlighting (#11413)

packages/babel-highlight/package.json

@@ -17,7 +17,7 @@
   "dependencies": {
     "@babel/helper-validator-identifier": "^7.10.4",
     "chalk": "^2.0.0",
-    "js-tokens": "^4.0.0"
+    "js-tokens": "^6.0.0"
   },
   "devDependencies": {
     "strip-ansi": "^4.0.0"

packages/babel-highlight/src/index.js

@@ -1,17 +1,30 @@
-import jsTokens, { matchToToken } from "js-tokens";
-import { isReservedWord, isKeyword } from "@babel/helper-validator-identifier";
+import jsTokens from "js-tokens";
+import {
+  isStrictReservedWord,
+  isKeyword,
+} from "@babel/helper-validator-identifier";
 import Chalk from "chalk";
 
+/**
+ * Names that are always allowed as identifiers, but also appear as keywords
+ * within certain syntactic productions.
+ *
+ * https://tc39.es/ecma262/#sec-keywords-and-reserved-words
+ *
+ * `target` has been omitted since it is very likely going to be a false
+ * positive.
+ */
+const sometimesKeywords = new Set(["as", "async", "from", "get", "of", "set"]);
+
 /**
  * Chalk styles for token types.
  */
 function getDefs(chalk) {
   return {
     keyword: chalk.cyan,
     capitalized: chalk.yellow,
-    jsx_tag: chalk.yellow,
+    jsxIdentifier: chalk.yellow,
     punctuator: chalk.yellow,
-    // bracket:  intentionally omitted.
     number: chalk.magenta,
     string: chalk.green,
     regex: chalk.magenta,
@@ -25,11 +38,6 @@ function getDefs(chalk) {
  */
 const NEWLINE = /\r\n|[\n\r\u2028\u2029]/;
 
-/**
- * RegExp to test for what seems to be a JSX tag name.
- */
-const JSX_TAG = /^[a-z][\w-]*$/i;
-
 /**
  * RegExp to test for the three types of brackets.
  */
@@ -38,57 +46,111 @@ const BRACKET = /^[()[\]{}]$/;
 /**
  * Get the type of token, specifying punctuator type.
  */
-function getTokenType(match) {
-  const [offset, text] = match.slice(-2);
-  const token = matchToToken(match);
-
-  if (token.type === "name") {
-    if (isKeyword(token.value) || isReservedWord(token.value)) {
-      return "keyword";
-    }
-
+function getTokenType(token) {
+  if (token.type === "IdentifierName") {
     if (
-      JSX_TAG.test(token.value) &&
-      (text[offset - 1] === "<" || text.substr(offset - 2, 2) == "</")
+      isKeyword(token.value) ||
+      isStrictReservedWord(token.value, true) ||
+      sometimesKeywords.has(token.value)
     ) {
-      return "jsx_tag";
+      return "keyword";
     }
 
     if (token.value[0] !== token.value[0].toLowerCase()) {
       return "capitalized";
     }
   }
 
-  if (token.type === "punctuator" && BRACKET.test(token.value)) {
-    return "bracket";
+  if (token.type === "Punctuator" && BRACKET.test(token.value)) {
+    return "uncolored";
   }
 
   if (
-    token.type === "invalid" &&
+    token.type === "Invalid" &&
     (token.value === "@" || token.value === "#")
   ) {
     return "punctuator";
   }
 
-  return token.type;
+  switch (token.type) {
+    case "NumericLiteral":
+      return "number";
+
+    case "StringLiteral":
+    case "JSXString":
+    case "NoSubstitutionTemplate":
+      return "string";
+
+    case "RegularExpressionLiteral":
+      return "regex";
+
+    case "Punctuator":
+    case "JSXPunctuator":
+      return "punctuator";
+
+    case "MultiLineComment":
+    case "SingleLineComment":
+      return "comment";
+
+    case "Invalid":
+    case "JSXInvalid":
+      return "invalid";
+
+    case "JSXIdentifier":
+      return "jsxIdentifier";
+
+    default:
+      return "uncolored";
+  }
+}
+
+/**
+ * Turn a string of JS into an array of objects.
+ */
+function* tokenize(text: string) {
+  for (const token of jsTokens(text, { jsx: true })) {
+    switch (token.type) {
+      case "TemplateHead":
+        yield { type: "string", value: token.value.slice(0, -2) };
+        yield { type: "punctuator", value: "${" };
+        break;
+
+      case "TemplateMiddle":
+        yield { type: "punctuator", value: "}" };
+        yield { type: "string", value: token.value.slice(1, -2) };
+        yield { type: "punctuator", value: "${" };
+        break;
+
+      case "TemplateTail":
+        yield { type: "punctuator", value: "}" };
+        yield { type: "string", value: token.value.slice(1) };
+        break;
+
+      default:
+        yield { type: getTokenType(token), value: token.value };
+    }
+  }
 }
 
 /**
  * Highlight `text` using the token definitions in `defs`.
  */
 function highlightTokens(defs: Object, text: string) {
-  return text.replace(jsTokens, function (...args) {
-    const type = getTokenType(args);
+  let highlighted = "";
+
+  for (const { type, value } of tokenize(text)) {
     const colorize = defs[type];
     if (colorize) {
-      return args[0]
+      highlighted += value
         .split(NEWLINE)
         .map(str => colorize(str))
         .join("\n");
     } else {
-      return args[0];
+      highlighted += value;
     }
-  });
+  }
+
+  return highlighted;
 }
 
 type Options = {