fix(parser): correctly parse record and tuple tokens (#13418)

* fix(parser): correctly parse token location for `#{` and `#[` * fix: `bar` tokens * fix: don't parse record and tuple pipe closing tokens as operators
babel · Jun 9, 2021 · a64d08c · a64d08c
1 parent 56db172
commit a64d08c
Show file tree

Hide file tree

Showing 13 changed files with 301 additions and 7 deletions.
diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js
@@ -486,14 +486,14 @@ export default class Tokenizer extends ParserErrors {
         );
       }
 
+      this.state.pos += 2;
       if (next === charCodes.leftCurlyBrace) {
         // #{
         this.finishToken(tt.braceHashL);
       } else {
         // #[
         this.finishToken(tt.bracketHashL);
       }
-      this.state.pos += 2;
     } else if (isIdentifierStart(next)) {
       ++this.state.pos;
       this.finishToken(tt.privateName, this.readWord1(next));
@@ -615,8 +615,8 @@ export default class Tokenizer extends ParserErrors {
             Errors.RecordExpressionBarIncorrectEndSyntaxType,
           );
         }
-
-        this.finishOp(tt.braceBarR, 2);
+        this.state.pos += 2;
+        this.finishToken(tt.braceBarR);
         return;
       }
 
@@ -631,8 +631,8 @@ export default class Tokenizer extends ParserErrors {
             Errors.TupleExpressionBarIncorrectEndSyntaxType,
           );
         }
-
-        this.finishOp(tt.bracketBarR, 2);
+        this.state.pos += 2;
+        this.finishToken(tt.bracketBarR);
         return;
       }
     }
@@ -812,8 +812,8 @@ export default class Tokenizer extends ParserErrors {
           }
 
           // [|
-          this.finishToken(tt.bracketBarL);
           this.state.pos += 2;
+          this.finishToken(tt.bracketBarL);
         } else {
           ++this.state.pos;
           this.finishToken(tt.bracketL);
@@ -836,8 +836,8 @@ export default class Tokenizer extends ParserErrors {
           }
 
           // {|
-          this.finishToken(tt.braceBarL);
           this.state.pos += 2;
+          this.finishToken(tt.braceBarL);
         } else {
           ++this.state.pos;
           this.finishToken(tt.braceL);

diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/input.js
@@ -0,0 +1 @@
+{||}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/options.json
@@ -0,0 +1,4 @@
+{
+  "plugins": [["recordAndTuple", { "syntaxType": "bar" }]],
+  "tokens": true
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json
@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+        "expression": {
+          "type": "RecordExpression",
+          "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+          "properties": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "{|",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "|}",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
+    }
+  ]
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/input.js
@@ -0,0 +1 @@
+#{}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/options.json
@@ -0,0 +1,4 @@
+{
+  "plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]],
+  "tokens": true
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json
@@ -0,0 +1,67 @@
+{
+  "type": "File",
+  "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+        "expression": {
+          "type": "RecordExpression",
+          "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+          "properties": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "#{",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "}",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null
+      },
+      "start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
+    }
+  ]
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/input.js
@@ -0,0 +1 @@
+[||]
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/options.json
@@ -0,0 +1,4 @@
+{
+  "plugins": [["recordAndTuple", { "syntaxType": "bar" }]],
+  "tokens": true
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json
@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+        "expression": {
+          "type": "TupleExpression",
+          "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}},
+          "elements": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "[|",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "|]",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}}
+    }
+  ]
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/input.js
@@ -0,0 +1 @@
+#[]
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/options.json
@@ -0,0 +1,4 @@
+{
+  "plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]],
+  "tokens": true
+}
diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json
@@ -0,0 +1,69 @@
+{
+  "type": "File",
+  "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+  "program": {
+    "type": "Program",
+    "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+    "sourceType": "script",
+    "interpreter": null,
+    "body": [
+      {
+        "type": "ExpressionStatement",
+        "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+        "expression": {
+          "type": "TupleExpression",
+          "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}},
+          "elements": []
+        }
+      }
+    ],
+    "directives": []
+  },
+  "tokens": [
+    {
+      "type": {
+        "label": "#[",
+        "beforeExpr": true,
+        "startsExpr": true,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}}
+    },
+    {
+      "type": {
+        "label": "]",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}}
+    },
+    {
+      "type": {
+        "label": "eof",
+        "beforeExpr": false,
+        "startsExpr": false,
+        "rightAssociative": false,
+        "isLoop": false,
+        "isAssign": false,
+        "prefix": false,
+        "postfix": false,
+        "binop": null,
+        "updateContext": null
+      },
+      "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}}
+    }
+  ]
+}