Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Twig: Improved tokenization (#3259)
  • Loading branch information
RunDevelopment committed Dec 7, 2021
1 parent 21a3c2d commit e03a7c2
Show file tree
Hide file tree
Showing 14 changed files with 576 additions and 270 deletions.
2 changes: 1 addition & 1 deletion components.js

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion components.json
Expand Up @@ -1350,7 +1350,7 @@
},
"twig": {
"title": "Twig",
"require": "markup",
"require": "markup-templating",
"owner": "brandonkelly"
},
"typescript": {
Expand Down
78 changes: 38 additions & 40 deletions components/prism-twig.js
@@ -1,46 +1,44 @@
Prism.languages.twig = {
'comment': /\{#[\s\S]*?#\}/,
'tag': {
pattern: /\{\{[\s\S]*?\}\}|\{%[\s\S]*?%\}/,
'comment': /^\{#[\s\S]*?#\}$/,

'tag-name': {
pattern: /(^\{%-?\s*)\w+/,
lookbehind: true,
alias: 'keyword'
},
'delimiter': {
pattern: /^\{[{%]-?|-?[%}]\}$/,
alias: 'punctuation'
},

'string': {
pattern: /("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,
inside: {
'ld': {
pattern: /^(?:\{\{-?|\{%-?\s*\w+)/,
inside: {
'punctuation': /^(?:\{\{|\{%)-?/,
'keyword': /\w+/
}
},
'rd': {
pattern: /-?(?:%\}|\}\})$/,
inside: {
'punctuation': /.+/
}
},
'string': {
pattern: /("|')(?:\\.|(?!\1)[^\\\r\n])*\1/,
inside: {
'punctuation': /^['"]|['"]$/
}
},
'keyword': /\b(?:even|if|odd)\b/,
'boolean': /\b(?:false|null|true)\b/,
'number': /\b0x[\dA-Fa-f]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][-+]?\d+)?/,
'operator': [
{
pattern: /(\s)(?:and|b-and|b-or|b-xor|ends with|in|is|matches|not|or|same as|starts with)(?=\s)/,
lookbehind: true
},
/[=<>]=?|!=|\*\*?|\/\/?|\?:?|[-+~%|]/
],
'property': /\b[a-zA-Z_]\w*\b/,
'punctuation': /[()\[\]{}:.,]/
'punctuation': /^['"]|['"]$/
}
},
'keyword': /\b(?:even|if|odd)\b/,
'boolean': /\b(?:false|null|true)\b/,
'number': /\b0x[\dA-Fa-f]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][-+]?\d+)?/,
'operator': [
{
pattern: /(\s)(?:and|b-and|b-or|b-xor|ends with|in|is|matches|not|or|same as|starts with)(?=\s)/,
lookbehind: true
},
/[=<>]=?|!=|\*\*?|\/\/?|\?:?|[-+~%|]/
],
'punctuation': /[()\[\]{}:.,]/
};

// The rest can be parsed as HTML
'other': {
// We want non-blank matches
pattern: /\S(?:[\s\S]*\S)?/,
inside: Prism.languages.markup
Prism.hooks.add('before-tokenize', function (env) {
if (env.language !== 'twig') {
return;
}
};

var pattern = /\{(?:#[\s\S]*?#|%[\s\S]*?%|\{[\s\S]*?\})\}/g;
Prism.languages['markup-templating'].buildPlaceholders(env, 'twig', pattern);
});

Prism.hooks.add('after-tokenize', function (env) {
Prism.languages['markup-templating'].tokenizePlaceholders(env, 'twig');
});
2 changes: 1 addition & 1 deletion components/prism-twig.min.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion plugins/autoloader/prism-autoloader.js
Expand Up @@ -153,7 +153,7 @@
"markup-templating"
],
"textile": "markup",
"twig": "markup",
"twig": "markup-templating",
"typescript": "javascript",
"v": "clike",
"vala": "clike",
Expand Down
2 changes: 1 addition & 1 deletion plugins/autoloader/prism-autoloader.min.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

14 changes: 7 additions & 7 deletions tests/languages/twig+pug/twig_inclusion.test
Expand Up @@ -5,15 +5,15 @@

[
["filter-atpl", [
["filter-name", ":atpl"],
["tag", [
["ld", [["punctuation", "{{"]]],
["number", "42"],
["rd", [["punctuation", "}}"]]]
]]
["filter-name", ":atpl"],

["punctuation", "{"],
["punctuation", "{"],
["number", "42"],
["delimiter", "}}"]
]]
]

----------------------------------------------------

Checks for atpl filter (Twig) in Jade.
Checks for atpl filter (Twig) in Jade.
20 changes: 10 additions & 10 deletions tests/languages/twig/boolean_feature.test
Expand Up @@ -5,23 +5,23 @@
----------------------------------------------------

[
["tag", [
["ld", [["punctuation", "{{"]]],
["twig", [
["delimiter", "{{"],
["boolean", "null"],
["rd", [["punctuation", "}}"]]]
["delimiter", "}}"]
]],
["tag", [
["ld", [["punctuation", "{{-"]]],
["twig", [
["delimiter", "{{-"],
["boolean", "true"],
["rd", [["punctuation", "-}}"]]]
["delimiter", "-}}"]
]],
["tag", [
["ld", [["punctuation", "{{"]]],
["twig", [
["delimiter", "{{"],
["boolean", "false"],
["rd", [["punctuation", "}}"]]]
["delimiter", "}}"]
]]
]

----------------------------------------------------

Checks for booleans and null.
Checks for booleans and null.
14 changes: 10 additions & 4 deletions tests/languages/twig/comment_feature.test
Expand Up @@ -6,11 +6,17 @@ bar #}
----------------------------------------------------

[
["comment", "{##}"],
["comment", "{# foo #}"],
["comment", "{# foo\r\nbar #}"]
["twig", [
["comment", "{##}"]
]],
["twig", [
["comment", "{# foo #}"]
]],
["twig", [
["comment", "{# foo\r\nbar #}"]
]]
]

----------------------------------------------------

Checks for comments.
Checks for comments.
73 changes: 44 additions & 29 deletions tests/languages/twig/keyword_feature.test
Expand Up @@ -6,48 +6,63 @@
----------------------------------------------------

[
["tag", [
["ld", [["punctuation", "{%"], ["keyword", "for"]]],
["property", "foo"],
["operator", "in"], ["property", "bar"],
["keyword", "if"], ["property", "baz"],
["rd", [["punctuation", "%}"]]]
["twig", [
["delimiter", "{%"],
["tag-name", "for"],
" foo ",
["operator", "in"],
" bar ",
["keyword", "if"],
" baz ",
["delimiter", "%}"]
]],
["tag", [
["ld", [["punctuation", "{%"], ["keyword", "endfor"]]],
["rd", [["punctuation", "%}"]]]
["twig", [
["delimiter", "{%"],
["tag-name", "endfor"],
["delimiter", "%}"]
]],
["tag", [
["ld", [["punctuation", "{%-"], ["keyword", "if"]]],
["property", "foo"], ["punctuation", "("], ["punctuation", ")"],
["rd", [["punctuation", "-%}"]]]

["twig", [
["delimiter", "{%-"],
["tag-name", "if"],
" foo",
["punctuation", "("],
["punctuation", ")"],
["delimiter", "-%}"]
]],
["tag", [
["ld", [["punctuation", "{%-"], ["keyword", "endif"]]],
["rd", [["punctuation", "-%}"]]]
["twig", [
["delimiter", "{%-"],
["tag-name", "endif"],
["delimiter", "-%}"]
]],
["tag", [
["ld", [["punctuation", "{%"], ["keyword", "macro"]]],
["property", "foobar"], ["punctuation", "("], ["punctuation", ")"],
["rd", [["punctuation", "%}"]]]

["twig", [
["delimiter", "{%"],
["tag-name", "macro"],
" foobar",
["punctuation", "("],
["punctuation", ")"],
["delimiter", "%}"]
]],
["tag", [
["ld", [["punctuation", "{%"], ["keyword", "endmacro"]]],
["rd", [["punctuation", "%}"]]]
["twig", [
["delimiter", "{%"],
["tag-name", "endmacro"],
["delimiter", "%}"]
]],
["tag", [
["ld", [["punctuation", "{{"]]],
["property", "foo"],

["twig", [
["delimiter", "{{"],
" foo ",
["operator", "is"],
["keyword", "even"],
["operator", "or"],
["property", "bar"],
" bar ",
["operator", "is"],
["keyword", "odd"],
["rd", [["punctuation", "}}"]]]
["delimiter", "}}"]
]]
]

----------------------------------------------------

Checks for keywords.
Checks for keywords.

0 comments on commit e03a7c2

Please sign in to comment.