Update js-tokens to 6.0.0

backport of 5b3ca43, part 2 Co-authored-by: Nicolò Ribaudo <nicolo.ribaudo@gmail.com>
babel · Jan 20, 2021 · d898c89 · d898c89
1 parent 1cde6df
commit d898c89
Show file tree

Hide file tree

Showing 3 changed files with 113 additions and 4 deletions.
diff --git a/packages/babel-highlight/package.json b/packages/babel-highlight/package.json
@@ -17,7 +17,7 @@
   "dependencies": {
     "@babel/helper-validator-identifier": "workspace:^7.10.4",
     "chalk": "^2.0.0",
-    "js-tokens": "^4.0.0"
+    "js-tokens": "condition:BABEL_8_BREAKING ? ^6.0.0 : ^4.0.0"
   },
   "devDependencies": {
     "strip-ansi": "^4.0.0"

diff --git a/packages/babel-highlight/src/index.js b/packages/babel-highlight/src/index.js
@@ -45,7 +45,99 @@ const BRACKET = /^[()[\]{}]$/;
 
 let tokenize;
 
-{
+if (process.env.BABEL_8_BREAKING) {
+  /**
+   * Get the type of token, specifying punctuator type.
+   */
+  const getTokenType = function (token) {
+    if (token.type === "IdentifierName") {
+      if (
+        isKeyword(token.value) ||
+        isStrictReservedWord(token.value, true) ||
+        sometimesKeywords.has(token.value)
+      ) {
+        return "keyword";
+      }
+
+      if (token.value[0] !== token.value[0].toLowerCase()) {
+        return "capitalized";
+      }
+    }
+
+    if (token.type === "Punctuator" && BRACKET.test(token.value)) {
+      return "uncolored";
+    }
+
+    if (
+      token.type === "Invalid" &&
+      (token.value === "@" || token.value === "#")
+    ) {
+      return "punctuator";
+    }
+
+    switch (token.type) {
+      case "NumericLiteral":
+        return "number";
+
+      case "StringLiteral":
+      case "JSXString":
+      case "NoSubstitutionTemplate":
+        return "string";
+
+      case "RegularExpressionLiteral":
+        return "regex";
+
+      case "Punctuator":
+      case "JSXPunctuator":
+        return "punctuator";
+
+      case "MultiLineComment":
+      case "SingleLineComment":
+        return "comment";
+
+      case "Invalid":
+      case "JSXInvalid":
+        return "invalid";
+
+      case "JSXIdentifier":
+        return "jsxIdentifier";
+
+      default:
+        return "uncolored";
+    }
+  };
+
+  /**
+   * Turn a string of JS into an array of objects.
+   */
+  tokenize = function* (text: string) {
+    for (const token of jsTokens(text, { jsx: true })) {
+      switch (token.type) {
+        case "TemplateHead":
+          yield { type: "string", value: token.value.slice(0, -2) };
+          yield { type: "punctuator", value: "${" };
+          break;
+
+        case "TemplateMiddle":
+          yield { type: "punctuator", value: "}" };
+          yield { type: "string", value: token.value.slice(1, -2) };
+          yield { type: "punctuator", value: "${" };
+          break;
+
+        case "TemplateTail":
+          yield { type: "punctuator", value: "}" };
+          yield { type: "string", value: token.value.slice(1) };
+          break;
+
+        default:
+          yield {
+            type: getTokenType(token),
+            value: token.value,
+          };
+      }
+    }
+  };
+} else {
   // This is only available in js-tokens@4, and not in js-tokens@6
   const { matchToToken } = jsTokensNs;
 

diff --git a/yarn.lock b/yarn.lock
@@ -858,7 +858,7 @@ __metadata:
   dependencies:
     "@babel/helper-validator-identifier": "workspace:^7.10.4"
     chalk: ^2.0.0
-    js-tokens: ^4.0.0
+    js-tokens: "condition:BABEL_8_BREAKING ? ^6.0.0 : ^4.0.0"
     strip-ansi: ^4.0.0
   languageName: unknown
   linkType: soft
@@ -9060,13 +9060,30 @@ fsevents@^1.2.7:
   languageName: node
   linkType: hard
 
-"js-tokens@npm:^4.0.0":
+"js-tokens-BABEL_8_BREAKING-false@npm:js-tokens@^4.0.0, js-tokens@npm:^4.0.0":
   version: 4.0.0
   resolution: "js-tokens@npm:4.0.0"
   checksum: 1fc4e4667ac2d972aba65148b9cbf9c17566b2394d3504238d8492bbd3e68f496c657eab06b26b40b17db5cac0a34d153a12130e2d2d2bb6dc2cdc8a4764eb1b
   languageName: node
   linkType: hard
 
+"js-tokens-BABEL_8_BREAKING-true@npm:js-tokens@^6.0.0":
+  version: 6.0.0
+  resolution: "js-tokens@npm:6.0.0"
+  checksum: 975859a4fd68cbaaabf106639df316e662b87b296afa9c6b00cfd25bc7642137433d18bf78e1e5578fc63c2a3e7334aad4fbed47f87c6c29f9a4f6760e79e322
+  languageName: node
+  linkType: hard
+
+"js-tokens@condition:BABEL_8_BREAKING ? ^6.0.0 : ^4.0.0":
+  version: 0.0.0-condition-e8d5a4
+  resolution: "js-tokens@condition:BABEL_8_BREAKING?^6.0.0:^4.0.0#e8d5a4"
+  dependencies:
+    js-tokens-BABEL_8_BREAKING-false: "npm:js-tokens@^4.0.0"
+    js-tokens-BABEL_8_BREAKING-true: "npm:js-tokens@^6.0.0"
+  checksum: 73bfc95ddbc9d7d95cc421d95803bc31e3c581798146d6711abdf43829dafeebb8d6c3b67676dc41535d28d528c105bb7a1932b85045fc33fe401e82ad4ae828
+  languageName: node
+  linkType: hard
+
 "js-yaml@npm:^3.13.1, js-yaml@npm:^3.2.1":
   version: 3.13.1
   resolution: "js-yaml@npm:3.13.1"