Structured text: Improved tokenization (#3213)

components/prism-iecst.js

@@ -3,6 +3,7 @@ Prism.languages.iecst = {
 		{
 			pattern: /(^|[^\\])(?:\/\*[\s\S]*?(?:\*\/|$)|\(\*[\s\S]*?(?:\*\)|$)|\{[\s\S]*?(?:\}|$))/,
 			lookbehind: true,
+			greedy: true,
 		},
 		{
 			pattern: /(^|[^\\:])\/\/.*/,
@@ -14,17 +15,18 @@ Prism.languages.iecst = {
 		pattern: /(["'])(?:\\(?:\r\n|[\s\S])|(?!\1)[^\\\r\n])*\1/,
 		greedy: true,
 	},
-	'class-name': /\b(?:END_)?(?:PROGRAM|CONFIGURATION|INTERFACE|FUNCTION_BLOCK|FUNCTION|ACTION|TRANSITION|TYPE|STRUCT|(?:INITIAL_)?STEP|NAMESPACE|LIBRARY|CHANNEL|FOLDER|RESOURCE|VAR_(?:ACCESS|CONFIG|EXTERNAL|GLOBAL|INPUT|IN_OUT|PUTPUT|TEMP)|VAR|METHOD|PROPERTY)\b/i,
-	'keyword': /\b(?:AT|BY|(?:END_)?(?:CASE|FOR|IF|REPEAT|WHILE)|CONSTANT|CONTINUE|DO|ELSE|ELSIF|EXIT|EXTENDS|FROM|GET|GOTO|IMPLEMENTS|JMP|NON_RETAIN|PRIVATE|PROTECTED|PUBLIC|RETAIN|RETURN|SET|TASK|THEN|TO|UNTIL|USING|WITH|__CATCH|__ENDTRY|__FINALLY|__TRY)\b/,
-	'variable': /\b(?:ANY|ARRAY|AT|BOOL|BYTE|U?(?:D|L|S)?INT|(?:D|L)?WORD|DATE(?:_AND_TIME)?|DT|L?REAL|POINTER|STRING|TIME(?:_OF_DAY)?|TOD)\b/,
-	'symbol': /%[IQM][XBWDL][\d.]*|%[IQ][\d.]*/,
+	'keyword': [
+		/\b(?:END_)?(?:PROGRAM|CONFIGURATION|INTERFACE|FUNCTION_BLOCK|FUNCTION|ACTION|TRANSITION|TYPE|STRUCT|(?:INITIAL_)?STEP|NAMESPACE|LIBRARY|CHANNEL|FOLDER|RESOURCE|VAR_(?:ACCESS|CONFIG|EXTERNAL|GLOBAL|INPUT|IN_OUT|OUTPUT|TEMP)|VAR|METHOD|PROPERTY)\b/i,
+		/\b(?:AT|BY|(?:END_)?(?:CASE|FOR|IF|REPEAT|WHILE)|CONSTANT|CONTINUE|DO|ELSE|ELSIF|EXIT|EXTENDS|FROM|GET|GOTO|IMPLEMENTS|JMP|NON_RETAIN|OF|PRIVATE|PROTECTED|PUBLIC|RETAIN|RETURN|SET|TASK|THEN|TO|UNTIL|USING|WITH|__CATCH|__ENDTRY|__FINALLY|__TRY)\b/
+	],
+	'class-name': /\b(?:ANY|ARRAY|BOOL|BYTE|U?(?:D|L|S)?INT|(?:D|L)?WORD|DATE(?:_AND_TIME)?|DT|L?REAL|POINTER|STRING|TIME(?:_OF_DAY)?|TOD)\b/,
+	'address': {
+		pattern: /%[IQM][XBWDL][\d.]*|%[IQ][\d.]*/,
+		alias: 'symbol'
+	},
 	'number': /\b(?:16#[\da-f]+|2#[01_]+|0x[\da-f]+)\b|\b(?:D|DT|T|TOD)#[\d_shmd:]*|\b[A-Z]*#[\d.,_]*|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[+-]?\d+)?/i,
 	'boolean': /\b(?:FALSE|NULL|TRUE)\b/,
-	'function': /\w+(?=\()/,
-	'operator': /(?:S?R?:?=>?|&&?|\*\*?|<=?|>=?|[-:^/+])|\b(?:AND|EQ|GE|GT|LE|LT|MOD|NE|NOT|OR|XOR)\b/,
-	'punctuation': /[();]/,
-	'type': {
-		'pattern': /#/,
-		'alias': 'selector',
-	},
+	'operator': /S?R?:?=>?|&&?|\*\*?|<[=>]?|>=?|[-:^/+#]|\b(?:AND|EQ|EXPT|GE|GT|LE|LT|MOD|NE|NOT|OR|XOR)\b/,
+	'function': /\b[a-z_]\w*(?=\s*\()/i,
+	'punctuation': /[()[\].,;]/,
 };

tests/languages/iecst/symbol.test → tests/languages/iecst/address_feature.test

@@ -5,18 +5,18 @@ END_VAR
 ----------------------------------------------------
 
 [
-	["class-name", "VAR"],
+	["keyword", "VAR"],
 
 	"\r\n    varname ",
 	["keyword", "AT"],
-	["symbol", "%QX1.0.0"],
+	["address", "%QX1.0.0"],
 	["operator", ":"],
-	["variable", "BOOL"],
+	["class-name", "BOOL"],
 	["operator", ":="],
 	["boolean", "TRUE"],
 	["punctuation", ";"],
 
-	["class-name", "END_VAR"]
+	["keyword", "END_VAR"]
 ]
 
 ----------------------------------------------------

tests/languages/iecst/operator_feature.test

@@ -0,0 +1,54 @@
+= <> < <= > >=
++ - * / ** ^ & &&
+
+: :=
+#
+
+AND
+EQ
+EXPT
+GE
+GT
+LE
+LT
+MOD
+NE
+NOT
+OR
+XOR
+
+----------------------------------------------------
+
+[
+	["operator", "="],
+	["operator", "<>"],
+	["operator", "<"],
+	["operator", "<="],
+	["operator", ">"],
+	["operator", ">="],
+
+	["operator", "+"],
+	["operator", "-"],
+	["operator", "*"],
+	["operator", "/"],
+	["operator", "**"],
+	["operator", "^"],
+	["operator", "&"],
+	["operator", "&&"],
+
+	["operator", ":"], ["operator", ":="],
+	["operator", "#"],
+
+	["operator", "AND"],
+	["operator", "EQ"],
+	["operator", "EXPT"],
+	["operator", "GE"],
+	["operator", "GT"],
+	["operator", "LE"],
+	["operator", "LT"],
+	["operator", "MOD"],
+	["operator", "NE"],
+	["operator", "NOT"],
+	["operator", "OR"],
+	["operator", "XOR"]
+]

tests/languages/iecst/punctuation_feature.test

@@ -0,0 +1,15 @@
+( ) [ ]
+, ; .
+
+----------------------------------------------------
+
+[
+	["punctuation", "("],
+	["punctuation", ")"],
+	["punctuation", "["],
+	["punctuation", "]"],
+
+	["punctuation", ","],
+	["punctuation", ";"],
+	["punctuation", "."]
+]