diff --git a/benchmark/babel-parser/many-nested-block-elements/bench.mjs b/benchmark/babel-parser/many-nested-block-elements/bench.mjs
new file mode 100644
index 000000000000..ee846834127c
--- /dev/null
+++ b/benchmark/babel-parser/many-nested-block-elements/bench.mjs
@@ -0,0 +1,22 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+ return "{".repeat(length) + "0" + "}".repeat(length);
+}
+function benchCases(name, implementation, options) {
+ for (const length of [128, 256, 512, 1024]) {
+ const input = createInput(length);
+ suite.add(`${name} ${length} nested template elements`, () => {
+ implementation.parse(input, options);
+ });
+ }
+}
+
+benchCases("baseline", baseline);
+benchCases("current", current);
+
+suite.on("cycle", report).run();
diff --git a/benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs b/benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs
new file mode 100644
index 000000000000..0d37ebf663df
--- /dev/null
+++ b/benchmark/babel-parser/many-nested-jsx-elements-with-attributes/bench.mjs
@@ -0,0 +1,25 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+ return "{y}".repeat(length) + "".repeat(length);
+}
+function benchCases(name, implementation, options) {
+ for (const length of [128, 256, 512, 1024]) {
+ const input = createInput(length);
+ suite.add(
+ `${name} ${length} nested jsx elements with one attribute and text`,
+ () => {
+ implementation.parse(input, options);
+ }
+ );
+ }
+}
+
+benchCases("baseline", baseline, { plugins: ["jsx"] });
+benchCases("current", current, { plugins: ["jsx"] });
+
+suite.on("cycle", report).run();
diff --git a/benchmark/babel-parser/many-nested-template-elements/bench.mjs b/benchmark/babel-parser/many-nested-template-elements/bench.mjs
new file mode 100644
index 000000000000..2fcaa8802a62
--- /dev/null
+++ b/benchmark/babel-parser/many-nested-template-elements/bench.mjs
@@ -0,0 +1,22 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+ return "` ${".repeat(length) + "0" + "}`".repeat(length);
+}
+function benchCases(name, implementation, options) {
+ for (const length of [128, 256, 512, 1024]) {
+ const input = createInput(length);
+ suite.add(`${name} ${length} nested template elements`, () => {
+ implementation.parse(input, options);
+ });
+ }
+}
+
+benchCases("baseline", baseline);
+benchCases("current", current);
+
+suite.on("cycle", report).run();
diff --git a/benchmark/babel-parser/many-template-elements/bench.mjs b/benchmark/babel-parser/many-template-elements/bench.mjs
new file mode 100644
index 000000000000..fc97e8e09088
--- /dev/null
+++ b/benchmark/babel-parser/many-template-elements/bench.mjs
@@ -0,0 +1,23 @@
+import Benchmark from "benchmark";
+import baseline from "@babel-baseline/parser";
+import current from "@babel/parser";
+import { report } from "../../util.mjs";
+
+const suite = new Benchmark.Suite();
+function createInput(length) {
+ return "`" + " ${0}".repeat(length) + "`";
+}
+function benchCases(name, implementation, options) {
+ for (const length of [128, 256, 512, 1024]) {
+ const input = createInput(length);
+ suite.add(`${name} ${length} template elements`, () => {
+ implementation.parse(input, options);
+ });
+ }
+}
+
+current.parse(createInput(1));
+benchCases("baseline", baseline);
+benchCases("current", current);
+
+suite.on("cycle", report).run();
diff --git a/eslint/babel-eslint-parser/src/convert/convertTokens.cjs b/eslint/babel-eslint-parser/src/convert/convertTokens.cjs
index a05a0a09824a..634410f04558 100644
--- a/eslint/babel-eslint-parser/src/convert/convertTokens.cjs
+++ b/eslint/babel-eslint-parser/src/convert/convertTokens.cjs
@@ -71,13 +71,6 @@ function convertTemplateType(tokens, tl) {
templateTokens.push(token);
break;
- case tl.eof:
- if (curlyBrace) {
- result.push(curlyBrace);
- }
-
- break;
-
default:
if (curlyBrace) {
result.push(curlyBrace);
@@ -186,6 +179,8 @@ function convertToken(token, source, tl) {
token.value = `${token.value}n`;
} else if (label === tl.privateName) {
token.type = "PrivateIdentifier";
+ } else if (label === tl.templateNonTail || label === tl.templateTail) {
+ token.type = "Template";
}
if (typeof token.type !== "string") {
@@ -196,12 +191,16 @@ function convertToken(token, source, tl) {
module.exports = function convertTokens(tokens, code, tl) {
const result = [];
-
- const withoutComments = convertTemplateType(tokens, tl).filter(
- t => t.type !== "CommentLine" && t.type !== "CommentBlock",
- );
- for (let i = 0, { length } = withoutComments; i < length; i++) {
- const token = withoutComments[i];
+ const templateTypeMergedTokens = process.env.BABEL_8_BREAKING
+ ? tokens
+ : convertTemplateType(tokens, tl);
+ // The last token is always tt.eof and should be skipped
+ for (let i = 0, { length } = templateTypeMergedTokens; i < length - 1; i++) {
+ const token = templateTypeMergedTokens[i];
+ const tokenType = token.type;
+ if (tokenType === "CommentLine" || tokenType === "CommentBlock") {
+ continue;
+ }
if (!process.env.BABEL_8_BREAKING) {
// Babel 8 already produces a single token
@@ -209,9 +208,9 @@ module.exports = function convertTokens(tokens, code, tl) {
if (
ESLINT_VERSION >= 8 &&
i + 1 < length &&
- token.type.label === tl.hash
+ tokenType.label === tl.hash
) {
- const nextToken = withoutComments[i + 1];
+ const nextToken = templateTypeMergedTokens[i + 1];
// We must disambiguate private identifier from the hack pipes topic token
if (nextToken.type.label === tl.name && token.end === nextToken.start) {
diff --git a/packages/babel-parser/src/parser/expression.js b/packages/babel-parser/src/parser/expression.js
index e221d4d978ff..abba6101bf0f 100644
--- a/packages/babel-parser/src/parser/expression.js
+++ b/packages/babel-parser/src/parser/expression.js
@@ -27,6 +27,7 @@ import {
tokenIsPostfix,
tokenIsPrefix,
tokenIsRightAssociative,
+ tokenIsTemplate,
tokenKeywordOrIdentifierIsKeyword,
tokenLabelName,
tokenOperatorPrecedence,
@@ -43,7 +44,7 @@ import {
isIdentifierStart,
canBeReservedWord,
} from "../util/identifier";
-import { Position } from "../util/location";
+import { Position, createPositionWithColumnOffset } from "../util/location";
import * as charCodes from "charcodes";
import {
BIND_OUTSIDE,
@@ -706,9 +707,10 @@ export default class ExpressionParser extends LValParser {
noCalls: ?boolean,
state: N.ParseSubscriptState,
): N.Expression {
- if (!noCalls && this.eat(tt.doubleColon)) {
+ const { type } = this.state;
+ if (!noCalls && type === tt.doubleColon) {
return this.parseBind(base, startPos, startLoc, noCalls, state);
- } else if (this.match(tt.backQuote)) {
+ } else if (tokenIsTemplate(type)) {
return this.parseTaggedTemplateExpression(
base,
startPos,
@@ -719,7 +721,7 @@ export default class ExpressionParser extends LValParser {
let optional = false;
- if (this.match(tt.questionDot)) {
+ if (type === tt.questionDot) {
if (noCalls && this.lookaheadCharCode() === charCodes.leftParenthesis) {
// stop at `?.` when parsing `new a?.()`
state.stop = true;
@@ -801,6 +803,7 @@ export default class ExpressionParser extends LValParser {
): N.Expression {
const node = this.startNodeAt(startPos, startLoc);
node.object = base;
+ this.next(); // eat '::'
node.callee = this.parseNoCallExpr();
state.stop = true;
return this.parseSubscripts(
@@ -1153,7 +1156,8 @@ export default class ExpressionParser extends LValParser {
case tt._new:
return this.parseNewOrNewTarget();
- case tt.backQuote:
+ case tt.templateNonTail:
+ case tt.templateTail:
return this.parseTemplate(false);
// BindExpression[Yield]
@@ -1831,37 +1835,47 @@ export default class ExpressionParser extends LValParser {
// Parse template expression.
parseTemplateElement(isTagged: boolean): N.TemplateElement {
- const elem = this.startNode();
- if (this.state.value === null) {
+ const { start, end, value } = this.state;
+ const elemStart = start + 1;
+ const elem = this.startNodeAt(
+ elemStart,
+ createPositionWithColumnOffset(this.state.startLoc, 1),
+ );
+ if (value === null) {
if (!isTagged) {
- this.raise(this.state.start + 1, Errors.InvalidEscapeSequenceTemplate);
+ this.raise(start + 2, Errors.InvalidEscapeSequenceTemplate);
}
}
+
+ const isTail = this.match(tt.templateTail);
+ const endOffset = isTail ? -1 : -2;
+ const elemEnd = end + endOffset;
elem.value = {
- raw: this.input
- .slice(this.state.start, this.state.end)
- .replace(/\r\n?/g, "\n"),
- cooked: this.state.value,
+ raw: this.input.slice(elemStart, elemEnd).replace(/\r\n?/g, "\n"),
+ cooked: value === null ? null : value.slice(1, endOffset),
};
+ elem.tail = isTail;
this.next();
- elem.tail = this.match(tt.backQuote);
- return this.finishNode(elem, "TemplateElement");
+ this.finishNode(elem, "TemplateElement");
+ this.resetEndLocation(
+ elem,
+ elemEnd,
+ createPositionWithColumnOffset(this.state.lastTokEndLoc, endOffset),
+ );
+ return elem;
}
// https://tc39.es/ecma262/#prod-TemplateLiteral
parseTemplate(isTagged: boolean): N.TemplateLiteral {
const node = this.startNode();
- this.next();
node.expressions = [];
let curElt = this.parseTemplateElement(isTagged);
node.quasis = [curElt];
while (!curElt.tail) {
- this.expect(tt.dollarBraceL);
node.expressions.push(this.parseTemplateSubstitution());
- this.expect(tt.braceR);
+ this.readTemplateContinuation();
node.quasis.push((curElt = this.parseTemplateElement(isTagged)));
}
- this.next();
return this.finishNode(node, "TemplateLiteral");
}
@@ -2680,21 +2694,22 @@ export default class ExpressionParser extends LValParser {
}
isAmbiguousAwait(): boolean {
+ if (this.hasPrecedingLineBreak()) return true;
+ const { type } = this.state;
return (
- this.hasPrecedingLineBreak() ||
// All the following expressions are ambiguous:
// await + 0, await - 0, await ( 0 ), await [ 0 ], await / 0 /u, await ``
- this.match(tt.plusMin) ||
- this.match(tt.parenL) ||
- this.match(tt.bracketL) ||
- this.match(tt.backQuote) ||
+ type === tt.plusMin ||
+ type === tt.parenL ||
+ type === tt.bracketL ||
+ tokenIsTemplate(type) ||
// Sometimes the tokenizer generates tt.slash for regexps, and this is
// handler by parseExprAtom
- this.match(tt.regexp) ||
- this.match(tt.slash) ||
+ type === tt.regexp ||
+ type === tt.slash ||
// This code could be parsed both as a modulo operator or as an intrinsic:
// await %x(0)
- (this.hasPlugin("v8intrinsic") && this.match(tt.modulo))
+ (this.hasPlugin("v8intrinsic") && type === tt.modulo)
);
}
diff --git a/packages/babel-parser/src/parser/statement.js b/packages/babel-parser/src/parser/statement.js
index 4addc19315ef..b21f7dfc5165 100644
--- a/packages/babel-parser/src/parser/statement.js
+++ b/packages/babel-parser/src/parser/statement.js
@@ -4,6 +4,7 @@ import * as N from "../types";
import {
tokenIsIdentifier,
tokenIsLoop,
+ tokenIsTemplate,
tt,
type TokenType,
getExportedToken,
@@ -39,7 +40,7 @@ import {
} from "../util/expression-scope";
import type { SourceType } from "../options";
import { Token } from "../tokenizer";
-import { Position } from "../util/location";
+import { createPositionWithColumnOffset } from "../util/location";
import { cloneStringLiteral, cloneIdentifier } from "./node";
const loopLabel = { kind: "loop" },
@@ -55,7 +56,10 @@ const loneSurrogate = /[\uD800-\uDFFF]/u;
const keywordRelationalOperator = /in(?:stanceof)?/y;
/**
- * Convert tt.privateName to tt.hash + tt.name for backward Babel 7 compat.
+ * Convert tokens for backward Babel 7 compat.
+ * tt.privateName => tt.hash + tt.name
+ * tt.templateTail => tt.backquote/tt.braceR + tt.template + tt.backquote
+ * tt.templateNonTail => tt.backquote/tt.braceR + tt.template + tt.dollarBraceL
* For performance reasons this routine mutates `tokens`, it is okay
* here since we execute `parseTopLevel` once for every file.
* @param {*} tokens
@@ -65,38 +69,116 @@ function babel7CompatTokens(tokens) {
for (let i = 0; i < tokens.length; i++) {
const token = tokens[i];
const { type } = token;
- if (type === tt.privateName) {
+ if (typeof type === "number") {
if (!process.env.BABEL_8_BREAKING) {
- const { loc, start, value, end } = token;
- const hashEndPos = start + 1;
- const hashEndLoc = new Position(loc.start.line, loc.start.column + 1);
- tokens.splice(
- i,
- 1,
- // $FlowIgnore: hacky way to create token
- new Token({
- type: getExportedToken(tt.hash),
- value: "#",
- start: start,
- end: hashEndPos,
- startLoc: loc.start,
- endLoc: hashEndLoc,
- }),
- // $FlowIgnore: hacky way to create token
- new Token({
- type: getExportedToken(tt.name),
- value: value,
- start: hashEndPos,
- end: end,
- startLoc: hashEndLoc,
- endLoc: loc.end,
- }),
- );
- i++;
- continue;
+ if (type === tt.privateName) {
+ const { loc, start, value, end } = token;
+ const hashEndPos = start + 1;
+ const hashEndLoc = createPositionWithColumnOffset(loc.start, 1);
+ tokens.splice(
+ i,
+ 1,
+ // $FlowIgnore: hacky way to create token
+ new Token({
+ type: getExportedToken(tt.hash),
+ value: "#",
+ start: start,
+ end: hashEndPos,
+ startLoc: loc.start,
+ endLoc: hashEndLoc,
+ }),
+ // $FlowIgnore: hacky way to create token
+ new Token({
+ type: getExportedToken(tt.name),
+ value: value,
+ start: hashEndPos,
+ end: end,
+ startLoc: hashEndLoc,
+ endLoc: loc.end,
+ }),
+ );
+ i++;
+ continue;
+ }
+
+ if (tokenIsTemplate(type)) {
+ const { loc, start, value, end } = token;
+ const backquoteEnd = start + 1;
+ const backquoteEndLoc = createPositionWithColumnOffset(loc.start, 1);
+ let startToken;
+ if (value.charCodeAt(0) === charCodes.graveAccent) {
+ // $FlowIgnore: hacky way to create token
+ startToken = new Token({
+ type: getExportedToken(tt.backQuote),
+ value: "`",
+ start: start,
+ end: backquoteEnd,
+ startLoc: loc.start,
+ endLoc: backquoteEndLoc,
+ });
+ } else {
+ // $FlowIgnore: hacky way to create token
+ startToken = new Token({
+ type: getExportedToken(tt.braceR),
+ value: "}",
+ start: start,
+ end: backquoteEnd,
+ startLoc: loc.start,
+ endLoc: backquoteEndLoc,
+ });
+ }
+ let templateValue,
+ templateElementEnd,
+ templateElementEndLoc,
+ endToken;
+ if (type === tt.templateTail) {
+ // ends with '`'
+ templateElementEnd = end - 1;
+ templateElementEndLoc = createPositionWithColumnOffset(loc.end, -1);
+ templateValue = value.slice(1, -1);
+ // $FlowIgnore: hacky way to create token
+ endToken = new Token({
+ type: getExportedToken(tt.backQuote),
+ value: "`",
+ start: templateElementEnd,
+ end: end,
+ startLoc: templateElementEndLoc,
+ endLoc: loc.end,
+ });
+ } else {
+ // ends with `${`
+ templateElementEnd = end - 2;
+ templateElementEndLoc = createPositionWithColumnOffset(loc.end, -2);
+ templateValue = value.slice(1, -2);
+ // $FlowIgnore: hacky way to create token
+ endToken = new Token({
+ type: getExportedToken(tt.dollarBraceL),
+ value: "${",
+ start: templateElementEnd,
+ end: end,
+ startLoc: templateElementEndLoc,
+ endLoc: loc.end,
+ });
+ }
+ tokens.splice(
+ i,
+ 1,
+ startToken,
+ // $FlowIgnore: hacky way to create token
+ new Token({
+ type: getExportedToken(tt.template),
+ value: templateValue,
+ start: backquoteEnd,
+ end: templateElementEnd,
+ startLoc: backquoteEndLoc,
+ endLoc: templateElementEndLoc,
+ }),
+ endToken,
+ );
+ i += 2;
+ continue;
+ }
}
- }
- if (typeof type === "number") {
// $FlowIgnore: we manipulate `token` for performance reasons
token.type = getExportedToken(type);
}
diff --git a/packages/babel-parser/src/plugins/flow/index.js b/packages/babel-parser/src/plugins/flow/index.js
index 660fcb593e14..da4b5a8bb80a 100644
--- a/packages/babel-parser/src/plugins/flow/index.js
+++ b/packages/babel-parser/src/plugins/flow/index.js
@@ -2807,11 +2807,9 @@ export default (superClass: Class): Class =>
// by parsing `jsxTagStart` to stop the JSX plugin from
// messing with the tokens
const { context } = this.state;
- const curContext = context[context.length - 1];
- if (curContext === tc.j_oTag) {
- context.length -= 2;
- } else if (curContext === tc.j_expr) {
- context.length -= 1;
+ const currentContext = context[context.length - 1];
+ if (currentContext === tc.j_oTag || currentContext === tc.j_expr) {
+ context.pop();
}
}
diff --git a/packages/babel-parser/src/plugins/jsx/index.js b/packages/babel-parser/src/plugins/jsx/index.js
index 682ad64ec904..d2ecb7306e88 100644
--- a/packages/babel-parser/src/plugins/jsx/index.js
+++ b/packages/babel-parser/src/plugins/jsx/index.js
@@ -46,12 +46,6 @@ const JsxErrors = makeErrorTemplates(
);
/* eslint-disable sort-keys */
-// Be aware that this file is always executed and not only when the plugin is enabled.
-// Therefore the contexts do always exist.
-tc.j_oTag = new TokContext("...", true);
-
function isFragment(object: ?N.JSXElement): boolean {
return object
? object.type === "JSXOpeningFragment" ||
@@ -301,8 +295,9 @@ export default (superClass: Class): Class =>
switch (this.state.type) {
case tt.braceL:
node = this.startNode();
+ this.setContext(tc.brace);
this.next();
- node = this.jsxParseExpressionContainer(node);
+ node = this.jsxParseExpressionContainer(node, tc.j_oTag);
if (node.expression.type === "JSXEmptyExpression") {
this.raise(node.start, JsxErrors.AttributeIsEmpty);
}
@@ -339,6 +334,7 @@ export default (superClass: Class): Class =>
jsxParseSpreadChild(node: N.JSXSpreadChild): N.JSXSpreadChild {
this.next(); // ellipsis
node.expression = this.parseExpression();
+ this.setContext(tc.j_oTag);
this.expect(tt.braceR);
return this.finishNode(node, "JSXSpreadChild");
@@ -348,6 +344,7 @@ export default (superClass: Class): Class =>
jsxParseExpressionContainer(
node: N.JSXExpressionContainer,
+ previousContext: TokContext,
): N.JSXExpressionContainer {
if (this.match(tt.braceR)) {
node.expression = this.jsxParseEmptyExpression();
@@ -368,6 +365,7 @@ export default (superClass: Class): Class =>
node.expression = expression;
}
+ this.setContext(previousContext);
this.expect(tt.braceR);
return this.finishNode(node, "JSXExpressionContainer");
@@ -377,9 +375,12 @@ export default (superClass: Class): Class =>
jsxParseAttribute(): N.JSXAttribute {
const node = this.startNode();
- if (this.eat(tt.braceL)) {
+ if (this.match(tt.braceL)) {
+ this.setContext(tc.brace);
+ this.next();
this.expect(tt.ellipsis);
node.argument = this.parseMaybeAssignAllowIn();
+ this.setContext(tc.j_oTag);
this.expect(tt.braceR);
return this.finishNode(node, "JSXSpreadAttribute");
}
@@ -464,11 +465,14 @@ export default (superClass: Class): Class =>
case tt.braceL: {
const node = this.startNode();
+ this.setContext(tc.brace);
this.next();
if (this.match(tt.ellipsis)) {
children.push(this.jsxParseSpreadChild(node));
} else {
- children.push(this.jsxParseExpressionContainer(node));
+ children.push(
+ this.jsxParseExpressionContainer(node, tc.j_expr),
+ );
}
break;
@@ -537,6 +541,11 @@ export default (superClass: Class): Class =>
return this.jsxParseElementAt(startPos, startLoc);
}
+ setContext(newContext: TokContext) {
+ const { context } = this.state;
+ context[context.length - 1] = newContext;
+ }
+
// ==================================
// Overrides
// ==================================
@@ -559,6 +568,11 @@ export default (superClass: Class): Class =>
}
}
+ skipSpace() {
+ const curContext = this.curContext();
+ if (!curContext.preserveSpace) super.skipSpace();
+ }
+
getTokenFromCode(code: number): void {
const context = this.curContext();
@@ -597,7 +611,6 @@ export default (superClass: Class): Class =>
}
updateContext(prevType: TokenType): void {
- super.updateContext(prevType);
const { context, type } = this.state;
if (type === tt.slash && prevType === tt.jsxTagStart) {
// do not consider JSX expr -> JSX open tag -> ... anymore
@@ -605,17 +618,16 @@ export default (superClass: Class): Class =>
context.splice(-2, 2, tc.j_cTag);
this.state.canStartJSXElement = false;
} else if (type === tt.jsxTagStart) {
- context.push(
- tc.j_expr, // treat as beginning of JSX expression
- tc.j_oTag, // start opening tag context
- );
+ // start opening tag context
+ context.push(tc.j_oTag);
} else if (type === tt.jsxTagEnd) {
- const out = context.pop();
+ const out = context[context.length - 1];
if ((out === tc.j_oTag && prevType === tt.slash) || out === tc.j_cTag) {
context.pop();
this.state.canStartJSXElement =
context[context.length - 1] === tc.j_expr;
} else {
+ this.setContext(tc.j_expr);
this.state.canStartJSXElement = true;
}
} else {
diff --git a/packages/babel-parser/src/plugins/typescript/index.js b/packages/babel-parser/src/plugins/typescript/index.js
index c749d838655f..8fb93cddda71 100644
--- a/packages/babel-parser/src/plugins/typescript/index.js
+++ b/packages/babel-parser/src/plugins/typescript/index.js
@@ -14,8 +14,9 @@ import {
tokenIsKeywordOrIdentifier,
tt,
type TokenType,
+ tokenIsTemplate,
} from "../../tokenizer/types";
-import { types as ct } from "../../tokenizer/context";
+import { types as tc } from "../../tokenizer/context";
import * as N from "../../types";
import type { Position } from "../../util/location";
import type Parser from "../../parser";
@@ -1071,7 +1072,8 @@ export default (superClass: Class): Class =>
}
return this.tsParseParenthesizedType();
- case tt.backQuote:
+ case tt.templateNonTail:
+ case tt.templateTail:
return this.tsParseTemplateLiteralType();
default: {
const { type } = this.state;
@@ -2196,7 +2198,7 @@ export default (superClass: Class): Class =>
}
return this.finishCallExpression(node, state.optionalChainMember);
- } else if (this.match(tt.backQuote)) {
+ } else if (tokenIsTemplate(this.state.type)) {
const result = this.parseTaggedTemplateExpression(
base,
startPos,
@@ -2872,14 +2874,13 @@ export default (superClass: Class): Class =>
/*:: invariant(jsx.node != null) */
if (!jsx.error) return jsx.node;
- // Remove `tc.j_expr` and `tc.j_oTag` from context added
+ // Remove `tc.j_expr` or `tc.j_oTag` from context added
// by parsing `jsxTagStart` to stop the JSX plugin from
// messing with the tokens
const { context } = this.state;
- if (context[context.length - 1] === ct.j_oTag) {
- context.length -= 2;
- } else if (context[context.length - 1] === ct.j_expr) {
- context.length -= 1;
+ const currentContext = context[context.length - 1];
+ if (currentContext === tc.j_oTag || currentContext === tc.j_expr) {
+ context.pop();
}
}
diff --git a/packages/babel-parser/src/tokenizer/context.js b/packages/babel-parser/src/tokenizer/context.js
index ebf4793afc33..6b985901814b 100644
--- a/packages/babel-parser/src/tokenizer/context.js
+++ b/packages/babel-parser/src/tokenizer/context.js
@@ -1,7 +1,7 @@
// @flow
-// The token context is used to track whether the apostrophe "`"
-// starts or ends a string template
+// The token context is used in JSX plugin to track
+// jsx tag / jsx text / normal JavaScript expression
export class TokContext {
constructor(token: string, preserveSpace?: boolean) {
@@ -13,9 +13,17 @@ export class TokContext {
preserveSpace: boolean;
}
-export const types: {
+const types: {
[key: string]: TokContext,
} = {
- brace: new TokContext("{"),
- template: new TokContext("`", true),
+ brace: new TokContext("{"), // normal JavaScript expression
+ j_oTag: new TokContext("...", true), // JSX expressions
};
+
+if (!process.env.BABEL_8_BREAKING) {
+ types.template = new TokContext("`", true);
+}
+
+export { types };
diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js
index 528cb051978b..9c70fe7ab5ea 100644
--- a/packages/babel-parser/src/tokenizer/index.js
+++ b/packages/babel-parser/src/tokenizer/index.js
@@ -13,7 +13,7 @@ import {
keywords as keywordTypes,
type TokenType,
} from "./types";
-import { type TokContext, types as ct } from "./context";
+import { type TokContext } from "./context";
import ParserErrors, { Errors, type ErrorTemplate } from "../parser/error";
import { SourceLocation } from "../util/location";
import {
@@ -296,8 +296,7 @@ export default class Tokenizer extends ParserErrors {
// properties.
nextToken(): void {
- const curContext = this.curContext();
- if (!curContext.preserveSpace) this.skipSpace();
+ this.skipSpace();
this.state.start = this.state.pos;
if (!this.isLookahead) this.state.startLoc = this.state.curPosition();
if (this.state.pos >= this.length) {
@@ -305,11 +304,7 @@ export default class Tokenizer extends ParserErrors {
return;
}
- if (curContext === ct.template) {
- this.readTmplToken();
- } else {
- this.getTokenFromCode(this.codePointAtPos(this.state.pos));
- }
+ this.getTokenFromCode(this.codePointAtPos(this.state.pos));
}
skipBlockComment(): N.CommentBlock | void {
@@ -921,8 +916,7 @@ export default class Tokenizer extends ParserErrors {
return;
case charCodes.graveAccent:
- ++this.state.pos;
- this.finishToken(tt.backQuote);
+ this.readTemplateToken();
return;
case charCodes.digit0: {
@@ -1375,36 +1369,40 @@ export default class Tokenizer extends ParserErrors {
this.finishToken(tt.string, out);
}
- // Reads template string tokens.
+ // Reads tempalte continuation `}...`
+ readTemplateContinuation(): void {
+ if (!this.match(tt.braceR)) {
+ this.unexpected(this.state.start, tt.braceR);
+ }
+ // rewind pos to `}`
+ this.state.pos--;
+ this.readTemplateToken();
+ }
- readTmplToken(): void {
+ // Reads template string tokens.
+ readTemplateToken(): void {
let out = "",
- chunkStart = this.state.pos,
+ chunkStart = this.state.pos, // eat '`' or `}`
containsInvalid = false;
+ ++this.state.pos; // eat '`' or `}`
for (;;) {
if (this.state.pos >= this.length) {
- throw this.raise(this.state.start, Errors.UnterminatedTemplate);
+ throw this.raise(this.state.start + 1, Errors.UnterminatedTemplate);
}
const ch = this.input.charCodeAt(this.state.pos);
+ if (ch === charCodes.graveAccent) {
+ ++this.state.pos; // eat '`'
+ out += this.input.slice(chunkStart, this.state.pos);
+ this.finishToken(tt.templateTail, containsInvalid ? null : out);
+ return;
+ }
if (
- ch === charCodes.graveAccent ||
- (ch === charCodes.dollarSign &&
- this.input.charCodeAt(this.state.pos + 1) ===
- charCodes.leftCurlyBrace)
+ ch === charCodes.dollarSign &&
+ this.input.charCodeAt(this.state.pos + 1) === charCodes.leftCurlyBrace
) {
- if (this.state.pos === this.state.start && this.match(tt.template)) {
- if (ch === charCodes.dollarSign) {
- this.state.pos += 2;
- this.finishToken(tt.dollarBraceL);
- return;
- } else {
- ++this.state.pos;
- this.finishToken(tt.backQuote);
- return;
- }
- }
+ this.state.pos += 2; // eat '${'
out += this.input.slice(chunkStart, this.state.pos);
- this.finishToken(tt.template, containsInvalid ? null : out);
+ this.finishToken(tt.templateNonTail, containsInvalid ? null : out);
return;
}
if (ch === charCodes.backslash) {
@@ -1633,44 +1631,7 @@ export default class Tokenizer extends ParserErrors {
}
}
- // the prevType is required by the jsx plugin
+ // updateContext is used by the jsx plugin
// eslint-disable-next-line no-unused-vars
- updateContext(prevType: TokenType): void {
- // Token-specific context update code
- // Note that we should avoid accessing `this.prodParam` in context update,
- // because it is executed immediately when last token is consumed, which may be
- // before `this.prodParam` is updated. e.g.
- // ```
- // function *g() { () => yield / 2 }
- // ```
- // When `=>` is eaten, the context update of `yield` is executed, however,
- // `this.prodParam` still has `[Yield]` production because it is not yet updated
- const { context, type } = this.state;
- switch (type) {
- case tt.braceR:
- context.pop();
- break;
- // we don't need to update context for tt.braceBarL because we do not pop context for tt.braceBarR
- // ideally only dollarBraceL "${" needs a non-template context
- // in order to indicate that the last "`" in `${`" starts a new string template
- // inside a template element within outer string template.
- // but when we popped such context in `}`, we lost track of whether this
- // `}` matches a `${` or other tokens matching `}`, so we have to push
- // such context in every token that `}` will match.
- case tt.braceL:
- case tt.braceHashL:
- case tt.dollarBraceL:
- context.push(ct.brace);
- break;
- case tt.backQuote:
- if (context[context.length - 1] === ct.template) {
- context.pop();
- } else {
- context.push(ct.template);
- }
- break;
- default:
- break;
- }
- }
+ updateContext(prevType: TokenType): void {}
}
diff --git a/packages/babel-parser/src/tokenizer/types.js b/packages/babel-parser/src/tokenizer/types.js
index 51539b05e077..7e3744f0da87 100644
--- a/packages/babel-parser/src/tokenizer/types.js
+++ b/packages/babel-parser/src/tokenizer/types.js
@@ -158,6 +158,10 @@ export const tt: { [name: string]: TokenType } = {
ellipsis: createToken("...", { beforeExpr }),
backQuote: createToken("`", { startsExpr }),
dollarBraceL: createToken("${", { beforeExpr, startsExpr }),
+ // start: isTemplate
+ templateTail: createToken("...`", { startsExpr }),
+ templateNonTail: createToken("...${", { beforeExpr, startsExpr }),
+ // end: isTemplate
at: createToken("@"),
hash: createToken("#", { startsExpr }),
@@ -402,6 +406,10 @@ export function tokenIsRightAssociative(token: TokenType): boolean {
return token === tt.exponent;
}
+export function tokenIsTemplate(token: TokenType): boolean {
+ return token >= tt.templateTail && token <= tt.templateNonTail;
+}
+
export function getExportedToken(token: TokenType): ExportedTokenType {
return tokenTypes[token];
}
diff --git a/packages/babel-parser/src/util/location.js b/packages/babel-parser/src/util/location.js
index 8e2ab59c5af9..c2bfd6f4ba9c 100644
--- a/packages/babel-parser/src/util/location.js
+++ b/packages/babel-parser/src/util/location.js
@@ -50,3 +50,22 @@ export function getLineInfo(input: string, offset: number): Position {
return new Position(line, offset - lineStart);
}
+
+/**
+ * creates a new position with a non-zero column offset from the given position.
+ * This function should be only be used when we create AST node out of the token
+ * boundaries, such as TemplateElement ends before tt.templateNonTail. This
+ * function does not skip whitespaces.
+ *
+ * @export
+ * @param {Position} position
+ * @param {number} columnOffset
+ * @returns {Position}
+ */
+export function createPositionWithColumnOffset(
+ position: Position,
+ columnOffset: number,
+) {
+ const { line, column } = position;
+ return new Position(line, column + columnOffset);
+}
diff --git a/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js
new file mode 100644
index 000000000000..43423443008c
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/input.js
@@ -0,0 +1 @@
+`${a}` // comment
diff --git a/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json
new file mode 100644
index 000000000000..e459344f740e
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/es2015/template/trailing-comments/output.json
@@ -0,0 +1,62 @@
+{
+ "type": "File",
+ "start":0,"end":17,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":17}},
+ "program": {
+ "type": "Program",
+ "start":0,"end":17,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":17}},
+ "sourceType": "script",
+ "interpreter": null,
+ "body": [
+ {
+ "type": "ExpressionStatement",
+ "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+ "expression": {
+ "type": "TemplateLiteral",
+ "start":0,"end":6,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":6}},
+ "expressions": [
+ {
+ "type": "Identifier",
+ "start":3,"end":4,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":4},"identifierName":"a"},
+ "name": "a"
+ }
+ ],
+ "quasis": [
+ {
+ "type": "TemplateElement",
+ "start":1,"end":1,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":1}},
+ "value": {
+ "raw": "",
+ "cooked": ""
+ },
+ "tail": false
+ },
+ {
+ "type": "TemplateElement",
+ "start":5,"end":5,"loc":{"start":{"line":1,"column":5},"end":{"line":1,"column":5}},
+ "value": {
+ "raw": "",
+ "cooked": ""
+ },
+ "tail": true
+ }
+ ]
+ },
+ "trailingComments": [
+ {
+ "type": "CommentLine",
+ "value": " comment",
+ "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17}}
+ }
+ ]
+ }
+ ],
+ "directives": []
+ },
+ "comments": [
+ {
+ "type": "CommentLine",
+ "value": " comment",
+ "start":7,"end":17,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":17}}
+ }
+ ]
+}
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js
new file mode 100644
index 000000000000..60abb3cb3104
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/input.js
@@ -0,0 +1,2 @@
+`before${x}middle${y}after`;
+`x`;
diff --git a/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json
new file mode 100644
index 000000000000..98c3eaf57999
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/basic/output.json
@@ -0,0 +1,345 @@
+{
+ "type": "File",
+ "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+ "program": {
+ "type": "Program",
+ "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+ "sourceType": "script",
+ "interpreter": null,
+ "body": [
+ {
+ "type": "ExpressionStatement",
+ "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+ "expression": {
+ "type": "TemplateLiteral",
+ "start":0,"end":27,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":27}},
+ "expressions": [
+ {
+ "type": "Identifier",
+ "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10},"identifierName":"x"},
+ "name": "x"
+ },
+ {
+ "type": "Identifier",
+ "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20},"identifierName":"y"},
+ "name": "y"
+ }
+ ],
+ "quasis": [
+ {
+ "type": "TemplateElement",
+ "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}},
+ "value": {
+ "raw": "before",
+ "cooked": "before"
+ },
+ "tail": false
+ },
+ {
+ "type": "TemplateElement",
+ "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}},
+ "value": {
+ "raw": "middle",
+ "cooked": "middle"
+ },
+ "tail": false
+ },
+ {
+ "type": "TemplateElement",
+ "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}},
+ "value": {
+ "raw": "after",
+ "cooked": "after"
+ },
+ "tail": true
+ }
+ ]
+ }
+ },
+ {
+ "type": "ExpressionStatement",
+ "start":29,"end":33,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":4}},
+ "expression": {
+ "type": "TemplateLiteral",
+ "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}},
+ "expressions": [],
+ "quasis": [
+ {
+ "type": "TemplateElement",
+ "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}},
+ "value": {
+ "raw": "x",
+ "cooked": "x"
+ },
+ "tail": true
+ }
+ ]
+ }
+ }
+ ],
+ "directives": []
+ },
+ "tokens": [
+ {
+ "type": {
+ "label": "`",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "`",
+ "start":0,"end":1,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":1}}
+ },
+ {
+ "type": {
+ "label": "template",
+ "beforeExpr": false,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "value": "before",
+ "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}}
+ },
+ {
+ "type": {
+ "label": "${",
+ "beforeExpr": true,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "${",
+ "start":7,"end":9,"loc":{"start":{"line":1,"column":7},"end":{"line":1,"column":9}}
+ },
+ {
+ "type": {
+ "label": "name",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "value": "x",
+ "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}}
+ },
+ {
+ "type": {
+ "label": "}",
+ "beforeExpr": true,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "}",
+ "start":10,"end":11,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":11}}
+ },
+ {
+ "type": {
+ "label": "template",
+ "beforeExpr": false,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "value": "middle",
+ "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}}
+ },
+ {
+ "type": {
+ "label": "${",
+ "beforeExpr": true,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "${",
+ "start":17,"end":19,"loc":{"start":{"line":1,"column":17},"end":{"line":1,"column":19}}
+ },
+ {
+ "type": {
+ "label": "name",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "value": "y",
+ "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20}}
+ },
+ {
+ "type": {
+ "label": "}",
+ "beforeExpr": true,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "}",
+ "start":20,"end":21,"loc":{"start":{"line":1,"column":20},"end":{"line":1,"column":21}}
+ },
+ {
+ "type": {
+ "label": "template",
+ "beforeExpr": false,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "value": "after",
+ "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}}
+ },
+ {
+ "type": {
+ "label": "`",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "`",
+ "start":26,"end":27,"loc":{"start":{"line":1,"column":26},"end":{"line":1,"column":27}}
+ },
+ {
+ "type": {
+ "label": ";",
+ "beforeExpr": true,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}}
+ },
+ {
+ "type": {
+ "label": "`",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "`",
+ "start":29,"end":30,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":1}}
+ },
+ {
+ "type": {
+ "label": "template",
+ "beforeExpr": false,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "value": "x",
+ "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}}
+ },
+ {
+ "type": {
+ "label": "`",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "`",
+ "start":31,"end":32,"loc":{"start":{"line":2,"column":2},"end":{"line":2,"column":3}}
+ },
+ {
+ "type": {
+ "label": ";",
+ "beforeExpr": true,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "start":32,"end":33,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4}}
+ },
+ {
+ "type": {
+ "label": "eof",
+ "beforeExpr": false,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null,
+ "updateContext": null
+ },
+ "start":33,"end":33,"loc":{"start":{"line":2,"column":4},"end":{"line":2,"column":4}}
+ }
+ ]
+}
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json
new file mode 100644
index 000000000000..9f5aa40a5dd6
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/tokens/template-string-babel-7/options.json
@@ -0,0 +1,4 @@
+{
+ "tokens": true,
+ "BABEL_8_BREAKING": false
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js b/packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js
new file mode 100644
index 000000000000..60abb3cb3104
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/tokens/template-string/basic/input.js
@@ -0,0 +1,2 @@
+`before${x}middle${y}after`;
+`x`;
diff --git a/packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json b/packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json
new file mode 100644
index 000000000000..a4670da0112b
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/tokens/template-string/basic/output.json
@@ -0,0 +1,216 @@
+{
+ "type": "File",
+ "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+ "program": {
+ "type": "Program",
+ "start":0,"end":33,"loc":{"start":{"line":1,"column":0},"end":{"line":2,"column":4}},
+ "sourceType": "script",
+ "interpreter": null,
+ "body": [
+ {
+ "type": "ExpressionStatement",
+ "start":0,"end":28,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":28}},
+ "expression": {
+ "type": "TemplateLiteral",
+ "start":0,"end":27,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":27}},
+ "expressions": [
+ {
+ "type": "Identifier",
+ "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10},"identifierName":"x"},
+ "name": "x"
+ },
+ {
+ "type": "Identifier",
+ "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20},"identifierName":"y"},
+ "name": "y"
+ }
+ ],
+ "quasis": [
+ {
+ "type": "TemplateElement",
+ "start":1,"end":7,"loc":{"start":{"line":1,"column":1},"end":{"line":1,"column":7}},
+ "value": {
+ "raw": "before",
+ "cooked": "before"
+ },
+ "tail": false
+ },
+ {
+ "type": "TemplateElement",
+ "start":11,"end":17,"loc":{"start":{"line":1,"column":11},"end":{"line":1,"column":17}},
+ "value": {
+ "raw": "middle",
+ "cooked": "middle"
+ },
+ "tail": false
+ },
+ {
+ "type": "TemplateElement",
+ "start":21,"end":26,"loc":{"start":{"line":1,"column":21},"end":{"line":1,"column":26}},
+ "value": {
+ "raw": "after",
+ "cooked": "after"
+ },
+ "tail": true
+ }
+ ]
+ }
+ },
+ {
+ "type": "ExpressionStatement",
+ "start":29,"end":33,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":4}},
+ "expression": {
+ "type": "TemplateLiteral",
+ "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}},
+ "expressions": [],
+ "quasis": [
+ {
+ "type": "TemplateElement",
+ "start":30,"end":31,"loc":{"start":{"line":2,"column":1},"end":{"line":2,"column":2}},
+ "value": {
+ "raw": "x",
+ "cooked": "x"
+ },
+ "tail": true
+ }
+ ]
+ }
+ }
+ ],
+ "directives": []
+ },
+ "tokens": [
+ {
+ "type": {
+ "label": "...${",
+ "beforeExpr": true,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "`before${",
+ "start":0,"end":9,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":9}}
+ },
+ {
+ "type": {
+ "label": "name",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "x",
+ "start":9,"end":10,"loc":{"start":{"line":1,"column":9},"end":{"line":1,"column":10}}
+ },
+ {
+ "type": {
+ "label": "...${",
+ "beforeExpr": true,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "}middle${",
+ "start":10,"end":19,"loc":{"start":{"line":1,"column":10},"end":{"line":1,"column":19}}
+ },
+ {
+ "type": {
+ "label": "name",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "y",
+ "start":19,"end":20,"loc":{"start":{"line":1,"column":19},"end":{"line":1,"column":20}}
+ },
+ {
+ "type": {
+ "label": "...`",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "}after`",
+ "start":20,"end":27,"loc":{"start":{"line":1,"column":20},"end":{"line":1,"column":27}}
+ },
+ {
+ "type": {
+ "label": ";",
+ "beforeExpr": true,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "start":27,"end":28,"loc":{"start":{"line":1,"column":27},"end":{"line":1,"column":28}}
+ },
+ {
+ "type": {
+ "label": "...`",
+ "beforeExpr": false,
+ "startsExpr": true,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "value": "`x`",
+ "start":29,"end":32,"loc":{"start":{"line":2,"column":0},"end":{"line":2,"column":3}}
+ },
+ {
+ "type": {
+ "label": ";",
+ "beforeExpr": true,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "start":32,"end":33,"loc":{"start":{"line":2,"column":3},"end":{"line":2,"column":4}}
+ },
+ {
+ "type": {
+ "label": "eof",
+ "beforeExpr": false,
+ "startsExpr": false,
+ "rightAssociative": false,
+ "isLoop": false,
+ "isAssign": false,
+ "prefix": false,
+ "postfix": false,
+ "binop": null
+ },
+ "start":33,"end":33,"loc":{"start":{"line":2,"column":4},"end":{"line":2,"column":4}}
+ }
+ ]
+}
\ No newline at end of file
diff --git a/packages/babel-parser/test/fixtures/tokens/template-string/options.json b/packages/babel-parser/test/fixtures/tokens/template-string/options.json
new file mode 100644
index 000000000000..5c34e72c636a
--- /dev/null
+++ b/packages/babel-parser/test/fixtures/tokens/template-string/options.json
@@ -0,0 +1,4 @@
+{
+ "tokens": true,
+ "BABEL_8_BREAKING": true
+}