From 0390e64420f090299b7bf62e65fecccd473d3ef5 Mon Sep 17 00:00:00 2001 From: Michael Schmidt Date: Tue, 7 Dec 2021 18:07:29 +0100 Subject: [PATCH] Pug: Improved filter tokenization (#3258) Co-authored-by: Jason Kurian --- components/prism-pug.js | 9 +++-- components/prism-pug.min.js | 2 +- .../coffeescript_inclusion.test | 12 ++++--- tests/languages/ejs+pug/ejs_inclusion.test | 29 +++++++-------- .../handlebars+pug/handlebars_inclusion.test | 6 ++-- tests/languages/less+pug/less_inclusion.test | 16 +++++---- .../livescript+pug/livescript_inclusion.test | 36 ++++++++++--------- .../markdown+pug/markdown_inclusion.test | 10 +++--- tests/languages/pug/filter_feature.test | 2 +- tests/languages/scss+pug/scss_inclusion.test | 10 +++--- .../stylus+pug/stylus_inclusion.test | 14 ++++---- tests/languages/twig+pug/twig_inclusion.test | 12 +++---- 12 files changed, 89 insertions(+), 69 deletions(-) diff --git a/components/prism-pug.js b/components/prism-pug.js index aa9d769c11..0afee3fcb7 100644 --- a/components/prism-pug.js +++ b/components/prism-pug.js @@ -33,7 +33,8 @@ 'filter-name': { pattern: /^:[\w-]+/, alias: 'variable' - } + }, + 'text': /\S[\s\S]*/, } }, @@ -172,7 +173,11 @@ pattern: /^:[\w-]+/, alias: 'variable' }, - rest: Prism.languages[filter.language] + 'text': { + pattern: /\S[\s\S]*/, + alias: [filter.language, 'language-' + filter.language], + inside: Prism.languages[filter.language] + } } }; } diff --git a/components/prism-pug.min.js b/components/prism-pug.min.js index 2e3939d8d1..4f2d7781e9 100644 --- a/components/prism-pug.min.js +++ b/components/prism-pug.min.js @@ -1 +1 @@ -!function(e){e.languages.pug={comment:{pattern:/(^([\t ]*))\/\/.*(?:(?:\r?\n|\r)\2[\t ].+)*/m,lookbehind:!0},"multiline-script":{pattern:/(^([\t ]*)script\b.*\.[\t ]*)(?:(?:\r?\n|\r(?!\n))(?:\2[\t ].+|\s*?(?=\r?\n|\r)))+/m,lookbehind:!0,inside:e.languages.javascript},filter:{pattern:/(^([\t ]*)):.+(?:(?:\r?\n|\r(?!\n))(?:\2[\t ].+|\s*?(?=\r?\n|\r)))+/m,lookbehind:!0,inside:{"filter-name":{pattern:/^:[\w-]+/,alias:"variable"}}},"multiline-plain-text":{pattern:/(^([\t ]*)[\w\-#.]+\.[\t ]*)(?:(?:\r?\n|\r(?!\n))(?:\2[\t ].+|\s*?(?=\r?\n|\r)))+/m,lookbehind:!0},markup:{pattern:/(^[\t ]*)<.+/m,lookbehind:!0,inside:e.languages.markup},doctype:{pattern:/((?:^|\n)[\t ]*)doctype(?: .+)?/,lookbehind:!0},"flow-control":{pattern:/(^[\t ]*)(?:case|default|each|else|if|unless|when|while)\b(?: .+)?/m,lookbehind:!0,inside:{each:{pattern:/^each .+? in\b/,inside:{keyword:/\b(?:each|in)\b/,punctuation:/,/}},branch:{pattern:/^(?:case|default|else|if|unless|when|while)\b/,alias:"keyword"},rest:e.languages.javascript}},keyword:{pattern:/(^[\t ]*)(?:append|block|extends|include|prepend)\b.+/m,lookbehind:!0},mixin:[{pattern:/(^[\t ]*)mixin .+/m,lookbehind:!0,inside:{keyword:/^mixin/,function:/\w+(?=\s*\(|\s*$)/,punctuation:/[(),.]/}},{pattern:/(^[\t ]*)\+.+/m,lookbehind:!0,inside:{name:{pattern:/^\+\w+/,alias:"function"},rest:e.languages.javascript}}],script:{pattern:/(^[\t ]*script(?:(?:&[^(]+)?\([^)]+\))*[\t ]).+/m,lookbehind:!0,inside:e.languages.javascript},"plain-text":{pattern:/(^[\t ]*(?!-)[\w\-#.]*[\w\-](?:(?:&[^(]+)?\([^)]+\))*\/?[\t ]).+/m,lookbehind:!0},tag:{pattern:/(^[\t ]*)(?!-)[\w\-#.]*[\w\-](?:(?:&[^(]+)?\([^)]+\))*\/?:?/m,lookbehind:!0,inside:{attributes:[{pattern:/&[^(]+\([^)]+\)/,inside:e.languages.javascript},{pattern:/\([^)]+\)/,inside:{"attr-value":{pattern:/(=\s*(?!\s))(?:\{[^}]*\}|[^,)\r\n]+)/,lookbehind:!0,inside:e.languages.javascript},"attr-name":/[\w-]+(?=\s*!?=|\s*[,)])/,punctuation:/[!=(),]+/}}],punctuation:/:/,"attr-id":/#[\w\-]+/,"attr-class":/\.[\w\-]+/}},code:[{pattern:/(^[\t ]*(?:-|!?=)).+/m,lookbehind:!0,inside:e.languages.javascript}],punctuation:/[.\-!=|]+/};for(var t=[{filter:"atpl",language:"twig"},{filter:"coffee",language:"coffeescript"},"ejs","handlebars","less","livescript","markdown",{filter:"sass",language:"scss"},"stylus"],n={},a=0,i=t.length;a(?:(?:\r?\n|\r(?!\n))(?:\\2[\t ].+|\\s*?(?=\r?\n|\r)))+".replace("",function(){return r.filter}),"m"),lookbehind:!0,inside:{"filter-name":{pattern:/^:[\w-]+/,alias:"variable"},rest:e.languages[r.language]}})}e.languages.insertBefore("pug","filter",n)}(Prism); \ No newline at end of file +!function(e){e.languages.pug={comment:{pattern:/(^([\t ]*))\/\/.*(?:(?:\r?\n|\r)\2[\t ].+)*/m,lookbehind:!0},"multiline-script":{pattern:/(^([\t ]*)script\b.*\.[\t ]*)(?:(?:\r?\n|\r(?!\n))(?:\2[\t ].+|\s*?(?=\r?\n|\r)))+/m,lookbehind:!0,inside:e.languages.javascript},filter:{pattern:/(^([\t ]*)):.+(?:(?:\r?\n|\r(?!\n))(?:\2[\t ].+|\s*?(?=\r?\n|\r)))+/m,lookbehind:!0,inside:{"filter-name":{pattern:/^:[\w-]+/,alias:"variable"},text:/\S[\s\S]*/}},"multiline-plain-text":{pattern:/(^([\t ]*)[\w\-#.]+\.[\t ]*)(?:(?:\r?\n|\r(?!\n))(?:\2[\t ].+|\s*?(?=\r?\n|\r)))+/m,lookbehind:!0},markup:{pattern:/(^[\t ]*)<.+/m,lookbehind:!0,inside:e.languages.markup},doctype:{pattern:/((?:^|\n)[\t ]*)doctype(?: .+)?/,lookbehind:!0},"flow-control":{pattern:/(^[\t ]*)(?:case|default|each|else|if|unless|when|while)\b(?: .+)?/m,lookbehind:!0,inside:{each:{pattern:/^each .+? in\b/,inside:{keyword:/\b(?:each|in)\b/,punctuation:/,/}},branch:{pattern:/^(?:case|default|else|if|unless|when|while)\b/,alias:"keyword"},rest:e.languages.javascript}},keyword:{pattern:/(^[\t ]*)(?:append|block|extends|include|prepend)\b.+/m,lookbehind:!0},mixin:[{pattern:/(^[\t ]*)mixin .+/m,lookbehind:!0,inside:{keyword:/^mixin/,function:/\w+(?=\s*\(|\s*$)/,punctuation:/[(),.]/}},{pattern:/(^[\t ]*)\+.+/m,lookbehind:!0,inside:{name:{pattern:/^\+\w+/,alias:"function"},rest:e.languages.javascript}}],script:{pattern:/(^[\t ]*script(?:(?:&[^(]+)?\([^)]+\))*[\t ]).+/m,lookbehind:!0,inside:e.languages.javascript},"plain-text":{pattern:/(^[\t ]*(?!-)[\w\-#.]*[\w\-](?:(?:&[^(]+)?\([^)]+\))*\/?[\t ]).+/m,lookbehind:!0},tag:{pattern:/(^[\t ]*)(?!-)[\w\-#.]*[\w\-](?:(?:&[^(]+)?\([^)]+\))*\/?:?/m,lookbehind:!0,inside:{attributes:[{pattern:/&[^(]+\([^)]+\)/,inside:e.languages.javascript},{pattern:/\([^)]+\)/,inside:{"attr-value":{pattern:/(=\s*(?!\s))(?:\{[^}]*\}|[^,)\r\n]+)/,lookbehind:!0,inside:e.languages.javascript},"attr-name":/[\w-]+(?=\s*!?=|\s*[,)])/,punctuation:/[!=(),]+/}}],punctuation:/:/,"attr-id":/#[\w\-]+/,"attr-class":/\.[\w\-]+/}},code:[{pattern:/(^[\t ]*(?:-|!?=)).+/m,lookbehind:!0,inside:e.languages.javascript}],punctuation:/[.\-!=|]+/};for(var t=[{filter:"atpl",language:"twig"},{filter:"coffee",language:"coffeescript"},"ejs","handlebars","less","livescript","markdown",{filter:"sass",language:"scss"},"stylus"],n={},a=0,i=t.length;a(?:(?:\r?\n|\r(?!\n))(?:\\2[\t ].+|\\s*?(?=\r?\n|\r)))+".replace("",function(){return r.filter}),"m"),lookbehind:!0,inside:{"filter-name":{pattern:/^:[\w-]+/,alias:"variable"},text:{pattern:/\S[\s\S]*/,alias:[r.language,"language-"+r.language],inside:e.languages[r.language]}}})}e.languages.insertBefore("pug","filter",n)}(Prism); \ No newline at end of file diff --git a/tests/languages/coffeescript+pug/coffeescript_inclusion.test b/tests/languages/coffeescript+pug/coffeescript_inclusion.test index ccf8963ba1..0ae21dc076 100644 --- a/tests/languages/coffeescript+pug/coffeescript_inclusion.test +++ b/tests/languages/coffeescript+pug/coffeescript_inclusion.test @@ -6,14 +6,16 @@ [ ["filter-coffee", [ ["filter-name", ":coffee"], - ["string", [ - "\"", - ["interpolation", "#{foo}"], - "\"" + ["text", [ + ["string", [ + "\"", + ["interpolation", "#{foo}"], + "\"" + ]] ]] ]] ] ---------------------------------------------------- -Checks for coffee filter (CoffeeScript) in Jade. \ No newline at end of file +Checks for coffee filter (CoffeeScript) in pug. diff --git a/tests/languages/ejs+pug/ejs_inclusion.test b/tests/languages/ejs+pug/ejs_inclusion.test index bbd9452970..f5b90b1bfd 100644 --- a/tests/languages/ejs+pug/ejs_inclusion.test +++ b/tests/languages/ejs+pug/ejs_inclusion.test @@ -6,19 +6,20 @@ [ ["filter-ejs", [ ["filter-name", ":ejs"], - ["language-javascript", [ - ["operator", "<"], - ["operator", "%"], - ["keyword", "var"], - " foo ", - ["operator", "="], - ["string", "''"], - ["punctuation", ","], - " bar ", - ["operator", "="], - ["boolean", "false"], - ["punctuation", ";"] - ]], - ["delimiter", "%>"] + ["text", [ + ["delimiter", "<%"], + ["language-javascript", [ + ["keyword", "var"], + " foo ", + ["operator", "="], + ["string", "''"], + ["punctuation", ","], + " bar ", + ["operator", "="], + ["boolean", "false"], + ["punctuation", ";"] + ]], + ["delimiter", "%>"] + ]] ]] ] diff --git a/tests/languages/handlebars+pug/handlebars_inclusion.test b/tests/languages/handlebars+pug/handlebars_inclusion.test index c69751fd06..156cf8a651 100644 --- a/tests/languages/handlebars+pug/handlebars_inclusion.test +++ b/tests/languages/handlebars+pug/handlebars_inclusion.test @@ -6,10 +6,12 @@ [ ["filter-handlebars", [ ["filter-name", ":handlebars"], - ["comment", "{{!comment}}"] + ["text", [ + ["comment", "{{!comment}}"] + ]] ]] ] ---------------------------------------------------- -Checks for handlebars filter in Jade. \ No newline at end of file +Checks for handlebars filter in pug. diff --git a/tests/languages/less+pug/less_inclusion.test b/tests/languages/less+pug/less_inclusion.test index 1cc8b350c8..ccd5f6a2d0 100644 --- a/tests/languages/less+pug/less_inclusion.test +++ b/tests/languages/less+pug/less_inclusion.test @@ -6,15 +6,17 @@ [ ["filter-less", [ ["filter-name", ":less"], - ["variable", [ - "@foo", - ["punctuation", ":"] - ]], - " #123", - ["punctuation", ";"] + ["text", [ + ["variable", [ + "@foo", + ["punctuation", ":"] + ]], + " #123", + ["punctuation", ";"] + ]] ]] ] ---------------------------------------------------- -Checks for less filter in Jade. \ No newline at end of file +Checks for less filter in pug. diff --git a/tests/languages/livescript+pug/livescript_inclusion.test b/tests/languages/livescript+pug/livescript_inclusion.test index ae2c878436..cc9bafedc0 100644 --- a/tests/languages/livescript+pug/livescript_inclusion.test +++ b/tests/languages/livescript+pug/livescript_inclusion.test @@ -6,23 +6,25 @@ [ ["filter-livescript", [ ["filter-name", ":livescript"], - ["interpolated-string", [ - ["string", "\""], - ["variable", "#foo"], - ["string", " "], - ["interpolation", [ - ["interpolation-punctuation", "#{"], - ["keyword", "if"], - ["regex", "/test/"], - ["operator", "=="], - ["string", "'test'"], - ["keyword", "then"], - ["number", "3"], - ["keyword", "else"], - ["number", "4"], - ["interpolation-punctuation", "}"] - ]], - ["string", "\""] + ["text", [ + ["interpolated-string", [ + ["string", "\""], + ["variable", "#foo"], + ["string", " "], + ["interpolation", [ + ["interpolation-punctuation", "#{"], + ["keyword", "if"], + ["regex", "/test/"], + ["operator", "=="], + ["string", "'test'"], + ["keyword", "then"], + ["number", "3"], + ["keyword", "else"], + ["number", "4"], + ["interpolation-punctuation", "}"] + ]], + ["string", "\""] + ]] ]] ]] ] diff --git a/tests/languages/markdown+pug/markdown_inclusion.test b/tests/languages/markdown+pug/markdown_inclusion.test index 01e6c3e796..cd3fb26278 100644 --- a/tests/languages/markdown+pug/markdown_inclusion.test +++ b/tests/languages/markdown+pug/markdown_inclusion.test @@ -6,13 +6,15 @@ [ ["filter-markdown", [ ["filter-name", ":markdown"], - ["title", [ - ["punctuation", "#"], - " title" + ["text", [ + ["title", [ + ["punctuation", "#"], + " title" + ]] ]] ]] ] ---------------------------------------------------- -Checks for markdown filter in Jade. \ No newline at end of file +Checks for markdown filter in pug. diff --git a/tests/languages/pug/filter_feature.test b/tests/languages/pug/filter_feature.test index 0e23802161..f055e20016 100644 --- a/tests/languages/pug/filter_feature.test +++ b/tests/languages/pug/filter_feature.test @@ -6,6 +6,6 @@ [ ["filter", [ ["filter-name", ":language"], - "\r\n\tcode" + ["text", "code"] ]] ] diff --git a/tests/languages/scss+pug/scss_inclusion.test b/tests/languages/scss+pug/scss_inclusion.test index 035147844b..194355d36a 100644 --- a/tests/languages/scss+pug/scss_inclusion.test +++ b/tests/languages/scss+pug/scss_inclusion.test @@ -6,12 +6,14 @@ [ ["filter-sass", [ ["filter-name", ":sass"], - ["keyword", "@extend"], - " .foo", - ["punctuation", ";"] + ["text", [ + ["keyword", "@extend"], + " .foo", + ["punctuation", ";"] + ]] ]] ] ---------------------------------------------------- -Checks for sass filter (Scss) in Jade. \ No newline at end of file +Checks for sass filter (Scss) in pug. diff --git a/tests/languages/stylus+pug/stylus_inclusion.test b/tests/languages/stylus+pug/stylus_inclusion.test index d154ab2e75..b00906d4b4 100644 --- a/tests/languages/stylus+pug/stylus_inclusion.test +++ b/tests/languages/stylus+pug/stylus_inclusion.test @@ -6,15 +6,17 @@ [ ["filter-stylus", [ ["filter-name", ":stylus"], - ["variable-declaration", [ - ["variable", "font-size"], - ["operator", "="], - ["number", "14"], - ["unit", "px"] + ["text", [ + ["variable-declaration", [ + ["variable", "font-size"], + ["operator", "="], + ["number", "14"], + ["unit", "px"] + ]] ]] ]] ] ---------------------------------------------------- -Checks for stylus filter in Jade. \ No newline at end of file +Checks for stylus filter in pug. diff --git a/tests/languages/twig+pug/twig_inclusion.test b/tests/languages/twig+pug/twig_inclusion.test index 3b84c61a7d..8066230fa9 100644 --- a/tests/languages/twig+pug/twig_inclusion.test +++ b/tests/languages/twig+pug/twig_inclusion.test @@ -6,14 +6,14 @@ [ ["filter-atpl", [ ["filter-name", ":atpl"], - - ["punctuation", "{"], - ["punctuation", "{"], - ["number", "42"], - ["delimiter", "}}"] + ["text", [ + ["delimiter", "{{"], + ["number", "42"], + ["delimiter", "}}"] + ]] ]] ] ---------------------------------------------------- -Checks for atpl filter (Twig) in Jade. +Checks for atpl filter (Twig) in pug.