From 9720b514176938e1f7cb6545876f02ee484840d2 Mon Sep 17 00:00:00 2001 From: rash Date: Mon, 14 Mar 2022 20:20:47 +0100 Subject: [PATCH 01/17] add support for custom template tokenizers --- src/common/parser-options.ts | 15 +++++++++++++++ src/html/parser.ts | 33 +++++++++++++++++++++++++++++++++ src/index.ts | 6 +++++- 3 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/common/parser-options.ts b/src/common/parser-options.ts index 3000bd76..8084dc10 100644 --- a/src/common/parser-options.ts +++ b/src/common/parser-options.ts @@ -1,7 +1,20 @@ import * as path from "path" +import type { IntermediateToken } from "../html/intermediate-tokenizer" import type { VDocumentFragment } from "../ast" import { getLang, isScriptElement, isScriptSetupElement } from "./ast-utils" +interface TemplateTokenizer { + nextToken(): IntermediateToken +} + +interface TemplateTokenizerConstructor { + new ( + templateText: string, + source: string, + options: { startingLine: number; startingColumn: number }, + ): TemplateTokenizer +} + export interface ParserOptions { // vue-eslint-parser options parser?: boolean | string @@ -37,6 +50,8 @@ export interface ParserOptions { // others // [key: string]: any + + templateTokenizer?: TemplateTokenizerConstructor } export function isSFCFile(parserOptions: ParserOptions) { diff --git a/src/html/parser.ts b/src/html/parser.ts index 68b9defb..d7f272e4 100644 --- a/src/html/parser.ts +++ b/src/html/parser.ts @@ -15,6 +15,7 @@ import type { VDocumentFragment, VElement, VExpressionContainer, + VLiteral, } from "../ast" import { NS, ParseError } from "../ast" import { debug } from "../common/debug" @@ -641,6 +642,38 @@ export class Parser { debug("[html] Text %j", token) const parent = this.currentNode + if (parent.type === "VElement" && parent.name === "template") { + const langAttribute = parent.startTag.attributes.find( + (a) => a.key.name === "lang", + ) + const lang = (langAttribute?.value as VLiteral).value + if ( + lang && + lang !== "html" && + this.baseParserOptions.templateTokenizer[lang] + ) { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const TemplateTokenizer = require(this.baseParserOptions + .templateTokenizer[lang]) + const templateTokenizer = new TemplateTokenizer( + token.value, + this.text, + { + startingLine: token.loc.start.line, + startingColumn: token.loc.start.column, + }, + ) + + let templateToken: IntermediateToken | null = null + while ( + (templateToken = templateTokenizer.nextToken()) != null + ) { + ;(this as any)[templateToken.type](templateToken) + } + // TODO integrate templateTokenizer.tokens/errors/comments + return + } + } parent.children.push({ type: "VText", range: token.range, diff --git a/src/index.ts b/src/index.ts index afff45e1..2b2ee3c1 100644 --- a/src/index.ts +++ b/src/index.ts @@ -43,6 +43,7 @@ function isVueFile(code: string, options: ParserOptions): boolean { * @param parserOptions The parser options. * @returns The parsing result. */ +// eslint-disable-next-line complexity export function parseForESLint( code: string, parserOptions: any, @@ -97,13 +98,16 @@ export function parseForESLint( const scripts = rootAST.children.filter(isScriptElement) const template = rootAST.children.find(isTemplateElement) const templateLang = getLang(template) || "html" + const hasTemplateTokenizer = + parserOptions.templateTokenizer[templateLang] const concreteInfo: AST.HasConcreteInfo = { tokens: rootAST.tokens, comments: rootAST.comments, errors: rootAST.errors, } const templateBody = - template != null && templateLang === "html" + template != null && + (templateLang === "html" || hasTemplateTokenizer) ? Object.assign(template, concreteInfo) : undefined From b2ae37f699de890bb2e7964c39a6db40faa6d41b Mon Sep 17 00:00:00 2001 From: rash Date: Mon, 14 Mar 2022 20:37:34 +0100 Subject: [PATCH 02/17] fix types --- src/common/parser-options.ts | 15 +-------------- src/html/parser.ts | 2 +- 2 files changed, 2 insertions(+), 15 deletions(-) diff --git a/src/common/parser-options.ts b/src/common/parser-options.ts index 8084dc10..5ed64d59 100644 --- a/src/common/parser-options.ts +++ b/src/common/parser-options.ts @@ -1,20 +1,7 @@ import * as path from "path" -import type { IntermediateToken } from "../html/intermediate-tokenizer" import type { VDocumentFragment } from "../ast" import { getLang, isScriptElement, isScriptSetupElement } from "./ast-utils" -interface TemplateTokenizer { - nextToken(): IntermediateToken -} - -interface TemplateTokenizerConstructor { - new ( - templateText: string, - source: string, - options: { startingLine: number; startingColumn: number }, - ): TemplateTokenizer -} - export interface ParserOptions { // vue-eslint-parser options parser?: boolean | string @@ -51,7 +38,7 @@ export interface ParserOptions { // others // [key: string]: any - templateTokenizer?: TemplateTokenizerConstructor + templateTokenizer?: { [key: string]: string } } export function isSFCFile(parserOptions: ParserOptions) { diff --git a/src/html/parser.ts b/src/html/parser.ts index d7f272e4..7eb06909 100644 --- a/src/html/parser.ts +++ b/src/html/parser.ts @@ -650,7 +650,7 @@ export class Parser { if ( lang && lang !== "html" && - this.baseParserOptions.templateTokenizer[lang] + this.baseParserOptions.templateTokenizer?.[lang] ) { // eslint-disable-next-line @typescript-eslint/no-require-imports const TemplateTokenizer = require(this.baseParserOptions From 2143d85a1ee11e3cd39a7c380df0f1b11fc3ec79 Mon Sep 17 00:00:00 2001 From: rash Date: Tue, 15 Mar 2022 10:02:11 +0100 Subject: [PATCH 03/17] Make existing tests pass --- src/html/parser.ts | 2 +- src/index.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/html/parser.ts b/src/html/parser.ts index 7eb06909..ae6b412a 100644 --- a/src/html/parser.ts +++ b/src/html/parser.ts @@ -646,7 +646,7 @@ export class Parser { const langAttribute = parent.startTag.attributes.find( (a) => a.key.name === "lang", ) - const lang = (langAttribute?.value as VLiteral).value + const lang = (langAttribute?.value as VLiteral)?.value if ( lang && lang !== "html" && diff --git a/src/index.ts b/src/index.ts index 2b2ee3c1..f83f33f5 100644 --- a/src/index.ts +++ b/src/index.ts @@ -99,7 +99,7 @@ export function parseForESLint( const template = rootAST.children.find(isTemplateElement) const templateLang = getLang(template) || "html" const hasTemplateTokenizer = - parserOptions.templateTokenizer[templateLang] + parserOptions?.templateTokenizer?.[templateLang] const concreteInfo: AST.HasConcreteInfo = { tokens: rootAST.tokens, comments: rootAST.comments, From 3b947fd5fac42e1faa8c4b0325baa23f01fd7668 Mon Sep 17 00:00:00 2001 From: rash Date: Tue, 15 Mar 2022 21:30:43 +0100 Subject: [PATCH 04/17] merge tokens into existing tokenizer --- src/html/parser.ts | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/html/parser.ts b/src/html/parser.ts index ae6b412a..9a8b580a 100644 --- a/src/html/parser.ts +++ b/src/html/parser.ts @@ -52,6 +52,8 @@ import { getScriptParser, getParserLangFromSFC, } from "../common/parser-options" +import sortedIndexBy from "lodash/sortedIndexBy" +import sortedLastIndexBy from "lodash/sortedLastIndexBy" const DIRECTIVE_NAME = /^(?:v-|[.:@#]).*[^.:@#]$/u const DT_DD = /^d[dt]$/u @@ -670,7 +672,24 @@ export class Parser { ) { ;(this as any)[templateToken.type](templateToken) } - // TODO integrate templateTokenizer.tokens/errors/comments + const index = sortedIndexBy( + this.tokenizer.tokens, + token, + (x) => x.range[0], + ) + const count = + sortedLastIndexBy( + this.tokenizer.tokens, + token, + (x) => x.range[1], + ) - index + this.tokenizer.tokens.splice( + index, + count, + ...templateTokenizer.tokens, + ) + this.tokenizer.comments.push(...templateTokenizer.comments) + this.tokenizer.errors.push(...templateTokenizer.errors) return } } From 9053d67d4cec80c3e973eb02e98737b9436245ee Mon Sep 17 00:00:00 2001 From: rash Date: Tue, 15 Mar 2022 21:31:02 +0100 Subject: [PATCH 05/17] add ast test --- .../ast/custom-template-tokenizer/ast.json | 902 ++++++++++++++++++ .../custom-tokenizer.js | 153 +++ .../parser-options.json | 5 + .../ast/custom-template-tokenizer/source.vue | 3 + .../token-ranges.json | 26 + .../ast/custom-template-tokenizer/tree.json | 107 +++ 6 files changed, 1196 insertions(+) create mode 100644 test/fixtures/ast/custom-template-tokenizer/ast.json create mode 100644 test/fixtures/ast/custom-template-tokenizer/custom-tokenizer.js create mode 100644 test/fixtures/ast/custom-template-tokenizer/parser-options.json create mode 100644 test/fixtures/ast/custom-template-tokenizer/source.vue create mode 100644 test/fixtures/ast/custom-template-tokenizer/token-ranges.json create mode 100644 test/fixtures/ast/custom-template-tokenizer/tree.json diff --git a/test/fixtures/ast/custom-template-tokenizer/ast.json b/test/fixtures/ast/custom-template-tokenizer/ast.json new file mode 100644 index 00000000..b3d647fe --- /dev/null +++ b/test/fixtures/ast/custom-template-tokenizer/ast.json @@ -0,0 +1,902 @@ +{ + "type": "Program", + "start": 0, + "end": 0, + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 0 + } + }, + "range": [ + 0, + 0 + ], + "body": [], + "sourceType": "script", + "comments": [], + "tokens": [], + "templateBody": { + "type": "VElement", + "range": [ + 0, + 114 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 3, + "column": 11 + } + }, + "name": "template", + "rawName": "template", + "namespace": "http://www.w3.org/1999/xhtml", + "startTag": { + "type": "VStartTag", + "range": [ + 0, + 28 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 28 + } + }, + "selfClosing": false, + "attributes": [ + { + "type": "VAttribute", + "range": [ + 10, + 27 + ], + "loc": { + "start": { + "line": 1, + "column": 10 + }, + "end": { + "line": 1, + "column": 27 + } + }, + "directive": false, + "key": { + "type": "VIdentifier", + "range": [ + 10, + 14 + ], + "loc": { + "start": { + "line": 1, + "column": 10 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "name": "lang", + "rawName": "lang" + }, + "value": { + "type": "VLiteral", + "range": [ + 15, + 27 + ], + "loc": { + "start": { + "line": 1, + "column": 15 + }, + "end": { + "line": 1, + "column": 27 + } + }, + "value": "customLang" + } + } + ] + }, + "children": [ + { + "type": "VElement", + "range": [ + 33, + 85 + ], + "loc": { + "start": { + "line": 2, + "column": 5 + }, + "end": { + "line": 2, + "column": 57 + } + }, + "name": "a-totally", + "rawName": "A-totally", + "namespace": "http://www.w3.org/1999/xhtml", + "startTag": { + "type": "VStartTag", + "range": [ + 33, + 54 + ], + "loc": { + "start": { + "line": 2, + "column": 5 + }, + "end": { + "line": 2, + "column": 26 + } + }, + "selfClosing": false, + "attributes": [ + { + "type": "VAttribute", + "directive": true, + "range": [ + 43, + 53 + ], + "loc": {}, + "key": { + "type": "VDirectiveKey", + "range": [ + 43, + 48 + ], + "loc": { + "start": { + "line": 2, + "column": 15 + }, + "end": { + "line": 2, + "column": 20 + } + }, + "name": { + "type": "VIdentifier", + "range": [ + 43, + 44 + ], + "loc": { + "start": { + "column": 15, + "line": 2 + }, + "end": { + "column": 16, + "line": 2 + } + }, + "name": "bind", + "rawName": ":" + }, + "argument": { + "type": "VIdentifier", + "range": [ + 44, + 48 + ], + "loc": { + "start": { + "column": 16, + "line": 2 + }, + "end": { + "column": 20, + "line": 2 + } + }, + "name": "made", + "rawName": "made" + }, + "modifiers": [] + }, + "value": { + "type": "VExpressionContainer", + "range": [ + 49, + 53 + ], + "loc": { + "start": { + "line": 2, + "column": 21 + }, + "end": { + "line": 2, + "column": 25 + } + }, + "expression": { + "type": "Identifier", + "start": 50, + "end": 52, + "loc": { + "start": { + "line": 2, + "column": 21 + }, + "end": { + "line": 2, + "column": 23 + } + }, + "range": [ + 50, + 52 + ], + "name": "up" + }, + "references": [ + { + "id": { + "type": "Identifier", + "start": 50, + "end": 52, + "loc": { + "start": { + "line": 2, + "column": 21 + }, + "end": { + "line": 2, + "column": 23 + } + }, + "range": [ + 50, + 52 + ], + "name": "up" + }, + "mode": "r" + } + ] + } + } + ] + }, + "children": [ + { + "type": "VExpressionContainer", + "range": [ + 54, + 85 + ], + "loc": { + "start": { + "line": 2, + "column": 26 + }, + "end": { + "line": 2, + "column": 57 + } + }, + "expression": { + "type": "BinaryExpression", + "start": 58, + "end": 79, + "loc": { + "start": { + "line": 2, + "column": 29 + }, + "end": { + "line": 2, + "column": 50 + } + }, + "range": [ + 58, + 79 + ], + "left": { + "type": "Identifier", + "start": 58, + "end": 68, + "loc": { + "start": { + "line": 2, + "column": 29 + }, + "end": { + "line": 2, + "column": 39 + } + }, + "range": [ + 58, + 68 + ], + "name": "templating" + }, + "operator": "+", + "right": { + "type": "Identifier", + "start": 71, + "end": 79, + "loc": { + "start": { + "line": 2, + "column": 42 + }, + "end": { + "line": 2, + "column": 50 + } + }, + "range": [ + 71, + 79 + ], + "name": "language" + } + }, + "references": [ + { + "id": { + "type": "Identifier", + "start": 58, + "end": 68, + "loc": { + "start": { + "line": 2, + "column": 29 + }, + "end": { + "line": 2, + "column": 39 + } + }, + "range": [ + 58, + 68 + ], + "name": "templating" + }, + "mode": "r" + }, + { + "id": { + "type": "Identifier", + "start": 71, + "end": 79, + "loc": { + "start": { + "line": 2, + "column": 42 + }, + "end": { + "line": 2, + "column": 50 + } + }, + "range": [ + 71, + 79 + ], + "name": "language" + }, + "mode": "r" + } + ] + } + ], + "endTag": { + "type": "VEndTag", + "range": [ + 85, + 85 + ], + "loc": { + "start": { + "line": 2, + "column": 57 + }, + "end": { + "line": 2, + "column": 57 + } + } + }, + "variables": [] + } + ], + "endTag": { + "type": "VEndTag", + "range": [ + 103, + 114 + ], + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 11 + } + } + }, + "variables": [], + "tokens": [ + { + "type": "HTMLTagOpen", + "range": [ + 0, + 9 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 1, + "column": 9 + } + }, + "value": "template" + }, + { + "type": "HTMLIdentifier", + "range": [ + 10, + 14 + ], + "loc": { + "start": { + "line": 1, + "column": 10 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "value": "lang" + }, + { + "type": "HTMLAssociation", + "range": [ + 14, + 15 + ], + "loc": { + "start": { + "line": 1, + "column": 14 + }, + "end": { + "line": 1, + "column": 15 + } + }, + "value": "" + }, + { + "type": "HTMLLiteral", + "range": [ + 15, + 27 + ], + "loc": { + "start": { + "line": 1, + "column": 15 + }, + "end": { + "line": 1, + "column": 27 + } + }, + "value": "customLang" + }, + { + "type": "HTMLTagClose", + "range": [ + 27, + 28 + ], + "loc": { + "start": { + "line": 1, + "column": 27 + }, + "end": { + "line": 1, + "column": 28 + } + }, + "value": "" + }, + { + "range": [ + 28, + 33 + ], + "loc": { + "start": { + "line": 1, + "column": 28 + }, + "end": { + "line": 2, + "column": 5 + } + }, + "value": "\n ", + "type": "CustomWhitespace" + }, + { + "range": [ + 33, + 43 + ], + "loc": { + "start": { + "line": 2, + "column": 5 + }, + "end": { + "line": 2, + "column": 15 + } + }, + "value": "A-totally", + "type": "CustomTagOpen" + }, + { + "type": "Punctuator", + "range": [ + 43, + 44 + ], + "loc": { + "start": { + "column": 15, + "line": 2 + }, + "end": { + "column": 16, + "line": 2 + } + }, + "value": ":" + }, + { + "type": "HTMLIdentifier", + "range": [ + 44, + 48 + ], + "loc": { + "start": { + "column": 16, + "line": 2 + }, + "end": { + "column": 20, + "line": 2 + } + }, + "value": "made" + }, + { + "range": [ + 48, + 49 + ], + "loc": { + "start": { + "line": 2, + "column": 20 + }, + "end": { + "line": 2, + "column": 21 + } + }, + "value": "", + "type": "CustomAssociation" + }, + { + "type": "Punctuator", + "range": [ + 49, + 50 + ], + "loc": { + "start": { + "line": 2, + "column": 20 + }, + "end": { + "line": 2, + "column": 21 + } + }, + "value": "\"" + }, + { + "type": "Identifier", + "value": "up", + "start": 50, + "end": 52, + "loc": { + "start": { + "line": 2, + "column": 21 + }, + "end": { + "line": 2, + "column": 23 + } + }, + "range": [ + 50, + 52 + ] + }, + { + "type": "Punctuator", + "range": [ + 52, + 53 + ], + "loc": { + "start": { + "line": 2, + "column": 23 + }, + "end": { + "line": 2, + "column": 24 + } + }, + "value": "\"" + }, + { + "range": [ + 53, + 54 + ], + "loc": { + "start": { + "line": 2, + "column": 25 + }, + "end": { + "line": 2, + "column": 26 + } + }, + "value": "", + "type": "CustomTagClose" + }, + { + "range": [ + 54, + 58 + ], + "loc": { + "start": { + "line": 2, + "column": 26 + }, + "end": { + "line": 2, + "column": 30 + } + }, + "value": "%%%", + "type": "VExpressionStart" + }, + { + "type": "Identifier", + "value": "templating", + "start": 58, + "end": 68, + "loc": { + "start": { + "line": 2, + "column": 29 + }, + "end": { + "line": 2, + "column": 39 + } + }, + "range": [ + 58, + 68 + ] + }, + { + "type": "Punctuator", + "value": "+", + "start": 69, + "end": 70, + "loc": { + "start": { + "line": 2, + "column": 40 + }, + "end": { + "line": 2, + "column": 41 + } + }, + "range": [ + 69, + 70 + ] + }, + { + "type": "Identifier", + "value": "language", + "start": 71, + "end": 79, + "loc": { + "start": { + "line": 2, + "column": 42 + }, + "end": { + "line": 2, + "column": 50 + } + }, + "range": [ + 71, + 79 + ] + }, + { + "range": [ + 81, + 85 + ], + "loc": { + "start": { + "line": 2, + "column": 53 + }, + "end": { + "line": 2, + "column": 57 + } + }, + "value": "%%%", + "type": "VExpressionEnd" + }, + { + "range": [ + 85, + 85 + ], + "loc": { + "start": { + "line": 2, + "column": 57 + }, + "end": { + "line": 2, + "column": 57 + } + }, + "value": "", + "type": "CustomEndTag" + }, + { + "type": "HTMLEndTagOpen", + "range": [ + 103, + 113 + ], + "loc": { + "start": { + "line": 3, + "column": 0 + }, + "end": { + "line": 3, + "column": 10 + } + }, + "value": "template" + }, + { + "type": "HTMLTagClose", + "range": [ + 113, + 114 + ], + "loc": { + "start": { + "line": 3, + "column": 10 + }, + "end": { + "line": 3, + "column": 11 + } + }, + "value": "" + }, + { + "type": "HTMLWhitespace", + "range": [ + 114, + 115 + ], + "loc": { + "start": { + "line": 3, + "column": 11 + }, + "end": { + "line": 4, + "column": 0 + } + }, + "value": "\n" + } + ], + "comments": [ + { + "range": [ + 85, + 102 + ], + "loc": { + "start": { + "line": 2, + "column": 57 + }, + "end": { + "line": 2, + "column": 74 + } + }, + "value": "language", + "type": "CustomComment" + } + ], + "errors": [ + { + "message": "abrupt-closing-of-empty-comment", + "index": 209, + "lineNumber": 15, + "column": 8 + } + ] + } +} \ No newline at end of file diff --git a/test/fixtures/ast/custom-template-tokenizer/custom-tokenizer.js b/test/fixtures/ast/custom-template-tokenizer/custom-tokenizer.js new file mode 100644 index 00000000..2faae046 --- /dev/null +++ b/test/fixtures/ast/custom-template-tokenizer/custom-tokenizer.js @@ -0,0 +1,153 @@ +const assert = require('assert') + +module.exports = class CustomTokenizer { + constructor (text, code, { startingLine, startingColumn }) { + // ignore actual input and just hardcode tokens + assert.equal(startingLine, 1) + assert.equal(startingColumn, 28) + this.line = startingLine + this.column = startingColumn + this.offset = 28 + this.tokens = [ + this.generateToken("\n ", { + type: "CustomWhitespace" + }), + this.generateToken("A-totally[", { + type: "CustomTagOpen", + value: "A-totally" + }), + this.generateToken(":made", { + type: "CustomIdentifier" + }), + this.generateToken("=", { + type: "CustomAssociation", + value: "" + }), + this.generateToken("\"up\"", { + type: "CustomLiteral", + value: "up" + }), + this.generateToken("]", { + type: "CustomTagClose", + value: "" + }), + this.generateToken(" %%%", { + type: "VExpressionStart", + value: "%%%" + }), + this.generateToken(" templating + language ", { + type: "CustomText", + value: "templating + language" + }), + this.generateToken("%%% ", { + type: "VExpressionEnd", + value: "%%%" + }), + this.generateToken("", { + type: "CustomEndTag", + value: "" + }), + this.generateToken("<=== comment ===>", { + type: "CustomComment", + value: "language" + }) + ] + this.comments = this.tokens.filter(token => token.type === "CustomComment") + this.tokens = this.tokens.filter(token => token.type !== "CustomComment") + + this.errors = [{ + "message": "totally-made-up-error", + "index": 9001, + "lineNumber": 15, + "column": 8 + }] + + const attribute = { + type: "VAttribute", + parent: {}, + directive: false, + range: [this.tokens[2].range[0], this.tokens[4].range[1]], + loc: { + start: this.tokens[2].start, + end: this.tokens[4].end + } + } + + attribute.key = { + type: "VIdentifier", + parent: attribute, + name: ":made", + rawName: ":made", + range: this.tokens[2].range, + loc: this.tokens[2].loc + } + + attribute.value = { + type: "VLiteral", + parent: attribute, + value: "up", + range: this.tokens[4].range, + loc: this.tokens[4].loc + } + + // these tokens get returned by nextToken + const intermediateTokens = [{ + type: "StartTag", + name: "a-totally", + rawName: "A-totally", + range: [this.tokens[1].range[0], this.tokens[5].range[1]], + loc: { + start: this.tokens[1].loc.start, + end: this.tokens[5].loc.end + }, + selfClosing: false, + attributes: [attribute] + }, { + type: "Mustache", + value: "templating + language", + range: [this.tokens[6].range[0], this.tokens[8].range[1]], + loc: { + start: this.tokens[6].loc.start, + end: this.tokens[8].loc.end + }, + startToken: this.tokens[6], + endToken: this.tokens[8] + }, { + type: "EndTag", + name: "a-totally", + range: this.tokens[9].range, + loc: this.tokens[9].loc, + }] + this.tokenIterator = intermediateTokens[Symbol.iterator]() + + console.log(this.tokens) + } + + nextToken () { + return this.tokenIterator.next().value + } + + // set range and loc based on text length and current offset + generateToken (text, data) { + const range = [this.offset, this.offset + text.length] + this.offset = range[1] + const loc = { + start: { + line: this.line, + column: this.column + } + } + this.line += text.split('\n').length - 1 + this.column = text.split('\n').length - 1 ? text.length - text.lastIndexOf('\n') : this.column + text.length + loc.end = { + line: this.line, + column: this.column + } + return { + range, + loc, + value: text, + ...data + } + } +} diff --git a/test/fixtures/ast/custom-template-tokenizer/parser-options.json b/test/fixtures/ast/custom-template-tokenizer/parser-options.json new file mode 100644 index 00000000..bfb18874 --- /dev/null +++ b/test/fixtures/ast/custom-template-tokenizer/parser-options.json @@ -0,0 +1,5 @@ +{ + "templateTokenizer": { + "customLang": "./test/fixtures/ast/custom-template-tokenizer/custom-tokenizer.js" + } +} diff --git a/test/fixtures/ast/custom-template-tokenizer/source.vue b/test/fixtures/ast/custom-template-tokenizer/source.vue new file mode 100644 index 00000000..7bfc47c0 --- /dev/null +++ b/test/fixtures/ast/custom-template-tokenizer/source.vue @@ -0,0 +1,3 @@ + diff --git a/test/fixtures/ast/custom-template-tokenizer/token-ranges.json b/test/fixtures/ast/custom-template-tokenizer/token-ranges.json new file mode 100644 index 00000000..a9994958 --- /dev/null +++ b/test/fixtures/ast/custom-template-tokenizer/token-ranges.json @@ -0,0 +1,26 @@ +[ + "", + "\n ", + "A-totally[", + ":", + "made", + "=", + "\"", + "up", + "\"", + "]", + " %%%", + " templatin", + " ", + " languag", + "%%% ", + "", + "", + "\n", + "<=== comment ===>" +] \ No newline at end of file diff --git a/test/fixtures/ast/custom-template-tokenizer/tree.json b/test/fixtures/ast/custom-template-tokenizer/tree.json new file mode 100644 index 00000000..c2c08b21 --- /dev/null +++ b/test/fixtures/ast/custom-template-tokenizer/tree.json @@ -0,0 +1,107 @@ +[ + { + "type": "VElement", + "text": "", + "children": [ + { + "type": "VStartTag", + "text": "", + "children": [] + } + ] + } +] \ No newline at end of file From bd0dc08403418e702d220e641ccdd933eb982fae Mon Sep 17 00:00:00 2001 From: rash Date: Tue, 15 Mar 2022 23:10:44 +0100 Subject: [PATCH 06/17] Write documentation for custom template tokenizers --- README.md | 20 ++++++++ ...implementing-custom-template-tokenizers.md | 46 +++++++++++++++++++ 2 files changed, 66 insertions(+) create mode 100644 docs/implementing-custom-template-tokenizers.md diff --git a/README.md b/README.md index 62f128eb..5ea351df 100644 --- a/README.md +++ b/README.md @@ -194,6 +194,26 @@ If set to `true`, to parse expressions in `v-bind` CSS functions inside `