From a64d08c101129eb171ca02aa4fc8effb24590580 Mon Sep 17 00:00:00 2001 From: Federico Ciardi Date: Wed, 9 Jun 2021 02:58:48 +0200 Subject: [PATCH] fix(parser): correctly parse record and tuple tokens (#13418) * fix(parser): correctly parse token location for `#{` and `#[` * fix: `bar` tokens * fix: don't parse record and tuple pipe closing tokens as operators --- packages/babel-parser/src/tokenizer/index.js | 14 ++-- .../record-and-tuple/brace-bar/input.js | 1 + .../record-and-tuple/brace-bar/options.json | 4 ++ .../record-and-tuple/brace-bar/output.json | 69 +++++++++++++++++++ .../record-and-tuple/brace-hash/input.js | 1 + .../record-and-tuple/brace-hash/options.json | 4 ++ .../record-and-tuple/brace-hash/output.json | 67 ++++++++++++++++++ .../record-and-tuple/bracket-bar/input.js | 1 + .../record-and-tuple/bracket-bar/options.json | 4 ++ .../record-and-tuple/bracket-bar/output.json | 69 +++++++++++++++++++ .../record-and-tuple/bracket-hash/input.js | 1 + .../bracket-hash/options.json | 4 ++ .../record-and-tuple/bracket-hash/output.json | 69 +++++++++++++++++++ 13 files changed, 301 insertions(+), 7 deletions(-) create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/input.js create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/options.json create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/input.js create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/options.json create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/input.js create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/options.json create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/input.js create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/options.json create mode 100644 packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json diff --git a/packages/babel-parser/src/tokenizer/index.js b/packages/babel-parser/src/tokenizer/index.js index b29621031f4a..f06a2e277205 100644 --- a/packages/babel-parser/src/tokenizer/index.js +++ b/packages/babel-parser/src/tokenizer/index.js @@ -486,6 +486,7 @@ export default class Tokenizer extends ParserErrors { ); } + this.state.pos += 2; if (next === charCodes.leftCurlyBrace) { // #{ this.finishToken(tt.braceHashL); @@ -493,7 +494,6 @@ export default class Tokenizer extends ParserErrors { // #[ this.finishToken(tt.bracketHashL); } - this.state.pos += 2; } else if (isIdentifierStart(next)) { ++this.state.pos; this.finishToken(tt.privateName, this.readWord1(next)); @@ -615,8 +615,8 @@ export default class Tokenizer extends ParserErrors { Errors.RecordExpressionBarIncorrectEndSyntaxType, ); } - - this.finishOp(tt.braceBarR, 2); + this.state.pos += 2; + this.finishToken(tt.braceBarR); return; } @@ -631,8 +631,8 @@ export default class Tokenizer extends ParserErrors { Errors.TupleExpressionBarIncorrectEndSyntaxType, ); } - - this.finishOp(tt.bracketBarR, 2); + this.state.pos += 2; + this.finishToken(tt.bracketBarR); return; } } @@ -812,8 +812,8 @@ export default class Tokenizer extends ParserErrors { } // [| - this.finishToken(tt.bracketBarL); this.state.pos += 2; + this.finishToken(tt.bracketBarL); } else { ++this.state.pos; this.finishToken(tt.bracketL); @@ -836,8 +836,8 @@ export default class Tokenizer extends ParserErrors { } // {| - this.finishToken(tt.braceBarL); this.state.pos += 2; + this.finishToken(tt.braceBarL); } else { ++this.state.pos; this.finishToken(tt.braceL); diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/input.js new file mode 100644 index 000000000000..58bca25f7eaf --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/input.js @@ -0,0 +1 @@ +{||} diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/options.json new file mode 100644 index 000000000000..616dcd299767 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/options.json @@ -0,0 +1,4 @@ +{ + "plugins": [["recordAndTuple", { "syntaxType": "bar" }]], + "tokens": true +} diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json new file mode 100644 index 000000000000..5c942aef4693 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-bar/output.json @@ -0,0 +1,69 @@ +{ + "type": "File", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "program": { + "type": "Program", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "expression": { + "type": "RecordExpression", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "properties": [] + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "{|", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}} + }, + { + "type": { + "label": "|}", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}} + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}} + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/input.js new file mode 100644 index 000000000000..feca1c1b895f --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/input.js @@ -0,0 +1 @@ +#{} diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/options.json new file mode 100644 index 000000000000..330c1681f890 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/options.json @@ -0,0 +1,4 @@ +{ + "plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]], + "tokens": true +} diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json new file mode 100644 index 000000000000..2902869a677a --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/brace-hash/output.json @@ -0,0 +1,67 @@ +{ + "type": "File", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "program": { + "type": "Program", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "expression": { + "type": "RecordExpression", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "properties": [] + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "#{", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}} + }, + { + "type": { + "label": "}", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null + }, + "start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}} + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}} + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/input.js new file mode 100644 index 000000000000..38030aa27cca --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/input.js @@ -0,0 +1 @@ +[||] diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/options.json new file mode 100644 index 000000000000..616dcd299767 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/options.json @@ -0,0 +1,4 @@ +{ + "plugins": [["recordAndTuple", { "syntaxType": "bar" }]], + "tokens": true +} diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json new file mode 100644 index 000000000000..4e4217e90f6d --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-bar/output.json @@ -0,0 +1,69 @@ +{ + "type": "File", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "program": { + "type": "Program", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "expression": { + "type": "TupleExpression", + "start":0,"end":4,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":4}}, + "elements": [] + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "[|", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}} + }, + { + "type": { + "label": "|]", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":2,"end":4,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":4}} + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":4,"end":4,"loc":{"start":{"line":1,"column":4},"end":{"line":1,"column":4}} + } + ] +} \ No newline at end of file diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/input.js b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/input.js new file mode 100644 index 000000000000..0fbee2942b3d --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/input.js @@ -0,0 +1 @@ +#[] diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/options.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/options.json new file mode 100644 index 000000000000..330c1681f890 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/options.json @@ -0,0 +1,4 @@ +{ + "plugins": [[ "recordAndTuple", { "syntaxType": "hash" }]], + "tokens": true +} diff --git a/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json new file mode 100644 index 000000000000..905fba8dd671 --- /dev/null +++ b/packages/babel-parser/test/fixtures/tokens/record-and-tuple/bracket-hash/output.json @@ -0,0 +1,69 @@ +{ + "type": "File", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "program": { + "type": "Program", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "sourceType": "script", + "interpreter": null, + "body": [ + { + "type": "ExpressionStatement", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "expression": { + "type": "TupleExpression", + "start":0,"end":3,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":3}}, + "elements": [] + } + } + ], + "directives": [] + }, + "tokens": [ + { + "type": { + "label": "#[", + "beforeExpr": true, + "startsExpr": true, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":0,"end":2,"loc":{"start":{"line":1,"column":0},"end":{"line":1,"column":2}} + }, + { + "type": { + "label": "]", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":2,"end":3,"loc":{"start":{"line":1,"column":2},"end":{"line":1,"column":3}} + }, + { + "type": { + "label": "eof", + "beforeExpr": false, + "startsExpr": false, + "rightAssociative": false, + "isLoop": false, + "isAssign": false, + "prefix": false, + "postfix": false, + "binop": null, + "updateContext": null + }, + "start":3,"end":3,"loc":{"start":{"line":1,"column":3},"end":{"line":1,"column":3}} + } + ] +} \ No newline at end of file