Smarty: Improved tokenization (#3268)

components.json

@@ -1219,6 +1219,7 @@
 		"smarty": {
 			"title": "Smarty",
 			"require": "markup-templating",
+			"optional": "php",
 			"owner": "Golmote"
 		},
 		"sml": {

components/prism-smarty.js

@@ -1,76 +1,120 @@
-/* TODO
-	Add support for variables inside double quoted strings
-	Add support for {php}
-*/
-
 (function (Prism) {
 
 	Prism.languages.smarty = {
-		'comment': /\{\*[\s\S]*?\*\}/,
+		'comment': {
+			pattern: /^\{\*[\s\S]*?\*\}/,
+			greedy: true
+		},
+		'embedded-php': {
+			pattern: /^\{php\}[\s\S]*?\{\/php\}/,
+			greedy: true,
+			inside: {
+				'smarty': {
+					pattern: /^\{php\}|\{\/php\}$/,
+					inside: null // see below
+				},
+				'php': {
+					pattern: /[\s\S]+/,
+					alias: 'language-php',
+					inside: Prism.languages.php
+				}
+			}
+		},
+		'string': [
+			{
+				pattern: /"(?:\\.|[^"\\\r\n])*"/,
+				greedy: true,
+				inside: {
+					'interpolation': {
+						pattern: /\{[^{}]*\}|`[^`]*`/,
+						inside: {
+							'interpolation-punctuation': {
+								pattern: /^[{`]|[`}]$/,
+								alias: 'punctuation'
+							},
+							'expression': {
+								pattern: /[\s\S]+/,
+								inside: null // see below
+							}
+						}
+					},
+					'variable': /\$\w+/
+				}
+			},
+			{
+				pattern: /'(?:\\.|[^'\\\r\n])*'/,
+				greedy: true
+			},
+		],
+		'keyword': {
+			pattern: /(^\{\/?)[a-z_]\w*\b(?!\()/i,
+			lookbehind: true,
+			greedy: true
+		},
 		'delimiter': {
-			pattern: /^\{|\}$/,
+			pattern: /^\{\/?|\}$/,
+			greedy: true,
 			alias: 'punctuation'
 		},
-		'string': /(["'])(?:\\.|(?!\1)[^\\\r\n])*\1/,
 		'number': /\b0x[\dA-Fa-f]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:[Ee][-+]?\d+)?/,
 		'variable': [
 			/\$(?!\d)\w+/,
 			/#(?!\d)\w+#/,
 			{
-				pattern: /(\.|->)(?!\d)\w+/,
+				pattern: /(\.|->|\w\s*=)(?!\d)\w+\b(?!\()/,
 				lookbehind: true
 			},
 			{
 				pattern: /(\[)(?!\d)\w+(?=\])/,
 				lookbehind: true
 			}
 		],
-		'function': [
-			{
-				pattern: /(\|\s*)@?(?!\d)\w+/,
-				lookbehind: true
-			},
-			/^\/?(?!\d)\w+/,
-			/(?!\d)\w+(?=\()/
-		],
-		'attr-name': {
-			// Value is made optional because it may have already been tokenized
-			pattern: /\w+\s*=\s*(?:(?!\d)\w+)?/,
-			inside: {
-				'variable': {
-					pattern: /(=\s*)(?!\d)\w+/,
-					lookbehind: true
-				},
-				'operator': /=/
-			}
+		'function': {
+			pattern: /(\|\s*)@?[a-z_]\w*|\b[a-z_]\w*(?=\()/i,
+			lookbehind: true
 		},
-		'punctuation': [
-			/[\[\]().,:`]|->/
-		],
+		'attr-name': /\b[a-z_]\w*(?=\s*=)/i,
+		'boolean': /\b(?:false|no|off|on|true|yes)\b/,
+		'punctuation': /[\[\](){}.,:`]|->/,
 		'operator': [
 			/[+\-*\/%]|==?=?|[!<>]=?|&&|\|\|?/,
 			/\bis\s+(?:not\s+)?(?:div|even|odd)(?:\s+by)?\b/,
 			/\b(?:and|eq|gt?e|gt|lt?e|lt|mod|neq?|not|or)\b/
-		],
-		'keyword': /\b(?:false|no|off|on|true|yes)\b/
+		]
 	};
 
+	Prism.languages.smarty['embedded-php'].inside.smarty.inside = Prism.languages.smarty;
+	Prism.languages.smarty.string[0].inside.interpolation.inside.expression.inside = Prism.languages.smarty;
+
+	var string = /"(?:\\.|[^"\\\r\n])*"|'(?:\\.|[^'\\\r\n])*'/;
+	var smartyPattern = RegExp(
+		// comments
+		/\{\*[\s\S]*?\*\}/.source +
+		'|' +
+		// php tags
+		/\{php\}[\s\S]*?\{\/php\}/.source +
+		'|' +
+		// smarty blocks
+		/\{(?:[^{}"']|<str>|\{(?:[^{}"']|<str>|\{(?:[^{}"']|<str>)*\})*\})*\}/.source
+			.replace(/<str>/g, function () { return string.source; }),
+		'g'
+	);
+
 	// Tokenize all inline Smarty expressions
 	Prism.hooks.add('before-tokenize', function (env) {
-		var smartyPattern = /\{\*[\s\S]*?\*\}|\{[\s\S]+?\}/g;
-		var smartyLitteralStart = '{literal}';
-		var smartyLitteralEnd = '{/literal}';
-		var smartyLitteralMode = false;
+		var smartyLiteralStart = '{literal}';
+		var smartyLiteralEnd = '{/literal}';
+		var smartyLiteralMode = false;
 
 		Prism.languages['markup-templating'].buildPlaceholders(env, 'smarty', smartyPattern, function (match) {
 			// Smarty tags inside {literal} block are ignored
-			if (match === smartyLitteralEnd) {
-				smartyLitteralMode = false;
+			if (match === smartyLiteralEnd) {
+				smartyLiteralMode = false;
 			}
 
-			if (!smartyLitteralMode) {
-				if (match === smartyLitteralStart) {
-					smartyLitteralMode = true;
+			if (!smartyLiteralMode) {
+				if (match === smartyLiteralStart) {
+					smartyLiteralMode = true;
 				}
 
 				return true;

tests/languages/smarty!+php/inclusion.test

@@ -0,0 +1,121 @@
+{php}
+   // including a php script directly from the template.
+   include('/path/to/display_weather.php');
+{/php}
+
+{* this template includes a {php} block that assign's the variable $varX *}
+{php}
+   global $foo, $bar;
+   if($foo == $bar){
+     echo 'This will be sent to browser';
+   }
+  // assign a variable to Smarty
+  $this->assign('varX','Toffee');
+{/php}
+{* output the variable *}
+<strong>{$varX}</strong> is my fav ice cream :-)
+
+----------------------------------------------------
+
+[
+	["smarty", [
+		["embedded-php", [
+			["smarty", [
+				["delimiter", "{"],
+				["keyword", "php"],
+				["delimiter", "}"]
+			]],
+			["php", [
+				["comment", "// including a php script directly from the template."],
+
+				["keyword", "include"],
+				["punctuation", "("],
+				["string", "'/path/to/display_weather.php'"],
+				["punctuation", ")"],
+				["punctuation", ";"]
+			]],
+			["smarty", [
+				["delimiter", "{/"],
+				["keyword", "php"],
+				["delimiter", "}"]
+			]]
+		]]
+	]],
+
+	["smarty", [
+		["comment", "{* this template includes a {php} block that assign's the variable $varX *}"]
+	]],
+
+	["smarty", [
+		["embedded-php", [
+			["smarty", [
+				["delimiter", "{"],
+				["keyword", "php"],
+				["delimiter", "}"]
+			]],
+			["php", [
+				["keyword", "global"],
+				["variable", "$foo"],
+				["punctuation", ","],
+				["variable", "$bar"],
+				["punctuation", ";"],
+
+				["keyword", "if"],
+				["punctuation", "("],
+				["variable", "$foo"],
+				["operator", "=="],
+				["variable", "$bar"],
+				["punctuation", ")"],
+				["punctuation", "{"],
+
+				["keyword", "echo"],
+				["string", "'This will be sent to browser'"],
+				["punctuation", ";"],
+
+				["punctuation", "}"],
+
+				["comment", "// assign a variable to Smarty"],
+
+				["variable", "$this"],
+				["operator", "->"],
+				["function", ["assign"]],
+				["punctuation", "("],
+				["string", "'varX'"],
+				["punctuation", ","],
+				["string", "'Toffee'"],
+				["punctuation", ")"],
+				["punctuation", ";"]
+			]],
+			["smarty", [
+				["delimiter", "{/"],
+				["keyword", "php"],
+				["delimiter", "}"]
+			]]
+		]]
+	]],
+
+	["smarty", [
+		["comment", "{* output the variable *}"]
+	]],
+
+	["tag", [
+		["tag", [
+			["punctuation", "<"],
+			"strong"
+		]],
+		["punctuation", ">"]
+	]],
+	["smarty", [
+		["delimiter", "{"],
+		["variable", "$varX"],
+		["delimiter", "}"]
+	]],
+	["tag", [
+		["tag", [
+			["punctuation", "</"],
+			"strong"
+		]],
+		["punctuation", ">"]
+	]],
+	" is my fav ice cream :-)"
+]

tests/languages/smarty/attr-name_feature.test

@@ -6,31 +6,25 @@
 [
 	["smarty", [
 		["delimiter", "{"],
-		["function", "assign"],
-		["attr-name", [
-			"var",
-			["operator", "="],
-			["variable", "foo"]
-		]],
-		["attr-name", [
-			"value",
-			["operator", "="]
-		]],
-		["string", "\"bar\""],
+		["keyword", "assign"],
+		["attr-name", "var"],
+		["operator", "="],
+		["variable", "foo"],
+		["attr-name", "value"],
+		["operator", "="],
+		["string", ["\"bar\""]],
 		["delimiter", "}"]
 	]],
 	["smarty", [
 		["delimiter", "{"],
-		["function", "foo"],
-		["attr-name", [
-			"bar ",
-			["operator", "="]
-		]],
+		["keyword", "foo"],
+		["attr-name", "bar"],
+		["operator", "="],
 		["number", "40"],
 		["delimiter", "}"]
 	]]
 ]
 
 ----------------------------------------------------
 
-Checks for attributes.
+Checks for attributes.

tests/languages/smarty/booelan_feature.test

@@ -0,0 +1,51 @@
+{if false}
+{if off}
+{if on}
+{if no}
+{if true}
+{if yes}
+
+----------------------------------------------------
+
+[
+	["smarty", [
+		["delimiter", "{"],
+		["keyword", "if"],
+		["boolean", "false"],
+		["delimiter", "}"]
+	]],
+	["smarty", [
+		["delimiter", "{"],
+		["keyword", "if"],
+		["boolean", "off"],
+		["delimiter", "}"]
+	]],
+	["smarty", [
+		["delimiter", "{"],
+		["keyword", "if"],
+		["boolean", "on"],
+		["delimiter", "}"]
+	]],
+	["smarty", [
+		["delimiter", "{"],
+		["keyword", "if"],
+		["boolean", "no"],
+		["delimiter", "}"]
+	]],
+	["smarty", [
+		["delimiter", "{"],
+		["keyword", "if"],
+		["boolean", "true"],
+		["delimiter", "}"]
+	]],
+	["smarty", [
+		["delimiter", "{"],
+		["keyword", "if"],
+		["boolean", "yes"],
+		["delimiter", "}"]
+	]]
+]
+
+----------------------------------------------------
+
+Checks for keywords.