Never throw for invalid escapes in tagged templates (#14964)

packages/babel-helper-string-parser/src/index.ts

@@ -59,7 +59,7 @@ export function readStringContents(
   const initialCurLine = curLine;
 
   let out = "";
-  let containsInvalid = false;
+  let firstInvalidLoc = null;
   let chunkStart = pos;
   const { length } = input;
   for (;;) {
@@ -75,25 +75,20 @@ export function readStringContents(
     }
     if (ch === charCodes.backslash) {
       out += input.slice(chunkStart, pos);
-      let escaped;
-      ({
-        ch: escaped,
-        pos,
-        lineStart,
-        curLine,
-      } = readEscapedChar(
+      const res = readEscapedChar(
         input,
         pos,
         lineStart,
         curLine,
         type === "template",
         errors,
-      ));
-      if (escaped === null) {
-        containsInvalid = true;
+      );
+      if (res.ch === null && !firstInvalidLoc) {
+        firstInvalidLoc = { pos, lineStart, curLine };
       } else {
-        out += escaped;
+        out += res.ch;
       }
+      ({ pos, lineStart, curLine } = res);
       chunkStart = pos;
     } else if (
       ch === charCodes.lineSeparator ||
@@ -121,7 +116,17 @@ export function readStringContents(
       ++pos;
     }
   }
-  return { pos, str: out, containsInvalid, lineStart, curLine };
+  return {
+    pos,
+    str: out,
+    firstInvalidLoc,
+    lineStart,
+    curLine,
+
+    // TODO(Babel 8): This is only needed for backwards compatibility,
+    // we can remove it.
+    containsInvalid: !!firstInvalidLoc,
+  };
 }
 
 function isStringEnd(
@@ -280,6 +285,7 @@ function readHexChar(
     forceLen,
     false,
     errors,
+    /* bailOnError */ !throwOnInvalid,
   ));
   if (n === null) {
     if (throwOnInvalid) {
@@ -322,6 +328,7 @@ export function readInt(
   forceLen: boolean,
   allowNumSeparator: boolean | "bail",
   errors: IntErrorHandlers,
+  bailOnError: boolean,
 ) {
   const start = pos;
   const forbiddenSiblings =
@@ -349,13 +356,15 @@ export function readInt(
       const next = input.charCodeAt(pos + 1);
 
       if (!allowNumSeparator) {
+        if (bailOnError) return { n: null, pos };
         errors.numericSeparatorInEscapeSequence(pos, lineStart, curLine);
       } else if (
         Number.isNaN(next) ||
         !isAllowedSibling(next) ||
         forbiddenSiblings.has(prev) ||
         forbiddenSiblings.has(next)
       ) {
+        if (bailOnError) return { n: null, pos };
         errors.unexpectedNumericSeparator(pos, lineStart, curLine);
       }
 
@@ -376,7 +385,12 @@ export function readInt(
     if (val >= radix) {
       // If we found a digit which is too big, errors.invalidDigit can return true to avoid
       // breaking the loop (this is used for error recovery).
-      if (val <= 9 && errors.invalidDigit(pos, lineStart, curLine, radix)) {
+      if (val <= 9 && bailOnError) {
+        return { n: null, pos };
+      } else if (
+        val <= 9 &&
+        errors.invalidDigit(pos, lineStart, curLine, radix)
+      ) {
         val = 0;
       } else if (forceLen) {
         val = 0;

packages/babel-parser/src/parser/expression.ts

@@ -2010,8 +2010,11 @@ export default abstract class ExpressionParser extends LValParser {
     if (value === null) {
       if (!isTagged) {
         this.raise(Errors.InvalidEscapeSequenceTemplate, {
-          // FIXME: explain
-          at: createPositionWithColumnOffset(startLoc, 2),
+          // FIXME: Adding 1 is probably wrong.
+          at: createPositionWithColumnOffset(
+            this.state.firstInvalidTemplateEscapePos,
+            1,
+          ),
         });
       }
     }

packages/babel-parser/src/tokenizer/index.ts

@@ -1131,6 +1131,7 @@ export default abstract class Tokenizer extends CommentsParser {
       forceLen,
       allowNumSeparator,
       this.errorHandlers_readInt,
+      /* bailOnError */ false,
     );
     this.state.pos = pos;
     return n;
@@ -1318,7 +1319,7 @@ export default abstract class Tokenizer extends CommentsParser {
   // Reads template string tokens.
   readTemplateToken(): void {
     const opening = this.input[this.state.pos];
-    const { str, containsInvalid, pos, curLine, lineStart } =
+    const { str, firstInvalidLoc, pos, curLine, lineStart } =
       readStringContents(
         "template",
         this.input,
@@ -1331,16 +1332,24 @@ export default abstract class Tokenizer extends CommentsParser {
     this.state.lineStart = lineStart;
     this.state.curLine = curLine;
 
+    if (firstInvalidLoc) {
+      this.state.firstInvalidTemplateEscapePos = new Position(
+        firstInvalidLoc.curLine,
+        firstInvalidLoc.pos - firstInvalidLoc.lineStart,
+        firstInvalidLoc.pos,
+      );
+    }
+
     if (this.input.codePointAt(pos) === charCodes.graveAccent) {
       this.finishToken(
         tt.templateTail,
-        containsInvalid ? null : opening + str + "`",
+        firstInvalidLoc ? null : opening + str + "`",
       );
     } else {
       this.state.pos++; // skip '{'
       this.finishToken(
         tt.templateNonTail,
-        containsInvalid ? null : opening + str + "${",
+        firstInvalidLoc ? null : opening + str + "${",
       );
     }
   }

packages/babel-parser/src/tokenizer/state.ts

@@ -135,6 +135,10 @@ export default class State {
   // escape sequences must not be interpreted as keywords.
   containsEsc: boolean = false;
 
+  // Used to track invalid escape sequences in template literals,
+  // that must be reported if the template is not tagged.
+  firstInvalidTemplateEscapePos: null | Position = null;
+
   // This property is used to track the following errors
   // - StrictNumericEscape
   // - StrictOctalLiteral

packages/babel-parser/test/fixtures/es2015/template/error-after-newline/output.json

@@ -2,7 +2,7 @@
   "type": "File",
   "start":0,"end":14,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":3,"column":2,"index":14}},
   "errors": [
-    "SyntaxError: Numeric separators are not allowed inside unicode escape sequences or hex escape sequences. (2:5)"
+    "SyntaxError: Invalid escape sequence in template. (2:1)"
   ],
   "program": {
     "type": "Program",
@@ -23,7 +23,7 @@
               "start":1,"end":12,"loc":{"start":{"line":1,"column":1,"index":1},"end":{"line":3,"column":0,"index":12}},
               "value": {
                 "raw": "\n\\u{12_34}\n",
-                "cooked": "\nሴ\n"
+                "cooked": null
               },
               "tail": true
             }

packages/babel-parser/test/fixtures/es2021/numeric-separator/template-with-invalid-numeric-separator-in-code-point-tagged/input.js

@@ -0,0 +1 @@
+tag`abc\u{1000_0000}`;

packages/babel-parser/test/fixtures/es2021/numeric-separator/template-with-invalid-numeric-separator-in-code-point-tagged/output.json

@@ -0,0 +1,42 @@
+{
+  "type": "File",
+  "start":0,"end":22,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":22,"index":22}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":22,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":22,"index":22}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":22,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":22,"index":22}},
+        "expression": {
+          "type": "TaggedTemplateExpression",
+          "start":0,"end":21,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":21,"index":21}},
+          "tag": {
+            "type": "Identifier",
+            "start":0,"end":3,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":3,"index":3},"identifierName":"tag"},
+            "name": "tag"
+          },
+          "quasi": {
+            "type": "TemplateLiteral",
+            "start":3,"end":21,"loc":{"start":{"line":1,"column":3,"index":3},"end":{"line":1,"column":21,"index":21}},
+            "expressions": [],
+            "quasis": [
+              {
+                "type": "TemplateElement",
+                "start":4,"end":20,"loc":{"start":{"line":1,"column":4,"index":4},"end":{"line":1,"column":20,"index":20}},
+                "value": {
+                  "raw": "abc\\u{1000_0000}",
+                  "cooked": null
+                },
+                "tail": true
+              }
+            ]
+          }
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-parser/test/fixtures/es2021/numeric-separator/template-with-invalid-numeric-separator-in-code-point/input.js

@@ -0,0 +1 @@
+`abc\u{1000_0000}`;

packages/babel-parser/test/fixtures/es2021/numeric-separator/template-with-invalid-numeric-separator-in-code-point/output.json

@@ -0,0 +1,36 @@
+{
+  "type": "File",
+  "start":0,"end":19,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":19,"index":19}},
+  "errors": [
+    "SyntaxError: Invalid escape sequence in template. (1:5)"
+  ],
+  "program": {
+    "type": "Program",
+    "start":0,"end":19,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":19,"index":19}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":19,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":19,"index":19}},
+        "expression": {
+          "type": "TemplateLiteral",
+          "start":0,"end":18,"loc":{"start":{"line":1,"column":0,"index":0},"end":{"line":1,"column":18,"index":18}},
+          "expressions": [],
+          "quasis": [
+            {
+              "type": "TemplateElement",
+              "start":1,"end":17,"loc":{"start":{"line":1,"column":1,"index":1},"end":{"line":1,"column":17,"index":17}},
+              "value": {
+                "raw": "abc\\u{1000_0000}",
+                "cooked": null
+              },
+              "tail": true
+            }
+          ]
+        }
+      }
+    ],
+    "directives": []
+  }
+}

packages/babel-types/src/definitions/core.ts

@@ -1979,40 +1979,33 @@ defineType("TemplateElement", {
         function templateElementCookedValidator(node: t.TemplateElement) {
           const raw = node.value.raw;
 
-          let str,
-            containsInvalid,
-            unterminatedCalled = false;
-          try {
-            const error = () => {
-              throw new Error();
-            };
-            ({ str, containsInvalid } = readStringContents(
-              "template",
-              raw,
-              0,
-              0,
-              0,
-              {
-                unterminated() {
-                  unterminatedCalled = true;
-                },
-                strictNumericEscape: error,
-                invalidEscapeSequence: error,
-                numericSeparatorInEscapeSequence: error,
-                unexpectedNumericSeparator: error,
-                invalidDigit: error,
-                invalidCodePoint: error,
+          let unterminatedCalled = false;
+
+          const error = () => {
+            // unreachable
+            throw new Error("Internal @babel/types error.");
+          };
+          const { str, firstInvalidLoc } = readStringContents(
+            "template",
+            raw,
+            0,
+            0,
+            0,
+            {
+              unterminated() {
+                unterminatedCalled = true;
               },
-            ));
-          } catch {
-            // TODO: When https://github.com/babel/babel/issues/14775 is fixed
-            // we can remove the try/catch block.
-            unterminatedCalled = true;
-            containsInvalid = true;
-          }
+              strictNumericEscape: error,
+              invalidEscapeSequence: error,
+              numericSeparatorInEscapeSequence: error,
+              unexpectedNumericSeparator: error,
+              invalidDigit: error,
+              invalidCodePoint: error,
+            },
+          );
           if (!unterminatedCalled) throw new Error("Invalid raw");
 
-          node.value.cooked = containsInvalid ? null : str;
+          node.value.cooked = firstInvalidLoc ? null : str;
         },
       ),
     },