Keyman: Improved tokenization (#3224)

components/prism-keyman.js

@@ -1,14 +1,44 @@
 Prism.languages.keyman = {
-	'comment': /\bc\s.*/i,
-	'function': /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|".+?"|'.+?')\s*\]/i,  // virtual key
-	'string': /("|').*?\1/,
-	'bold': [   // header statements, system stores and variable system stores
-		/&(?:baselayout|bitmap|capsalwaysoff|capsononly|copyright|ethnologuecode|hotkey|includecodes|keyboardversion|kmw_embedcss|kmw_embedjs|kmw_helpfile|kmw_helptext|kmw_rtl|language|layer|layoutfile|message|mnemoniclayout|name|oldcharposmatching|platform|shiftfreescaps|targets|version|visualkeyboard|windowslanguages)\b/i,
-		/\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i
-	],
-	'keyword': /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i,  // rule keywords
-	'atrule': /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i,   // structural keywords
-	'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i, // U+####, x###, d### characters and numbers
-	'operator': /[+>\\,()]/,
-	'tag': /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i   // prefixes
+	'comment': {
+		pattern: /\bc .*/i,
+		greedy: true
+	},
+	'string': {
+		pattern: /"[^"\r\n]*"|'[^'\r\n]*'/,
+		greedy: true
+	},
+	'virtual-key': {
+		pattern: /\[\s*(?:(?:ALT|CAPS|CTRL|LALT|LCTRL|NCAPS|RALT|RCTRL|SHIFT)\s+)*(?:[TKU]_[\w?]+|[A-E]\d\d?|"[^"\r\n]*"|'[^'\r\n]*')\s*\]/i,
+		greedy: true,
+		alias: 'function' // alias for styles
+	},
+
+	// https://help.keyman.com/developer/language/guide/headers
+	'header-keyword': {
+		pattern: /&\w+/,
+		alias: 'bold' // alias for styles
+	},
+	'header-statement': {
+		pattern: /\b(?:bitmap|bitmaps|caps always off|caps on only|copyright|hotkey|language|layout|message|name|shift frees caps|version)\b/i,
+		alias: 'bold' // alias for styles
+	},
+
+	'rule-keyword': {
+		pattern: /\b(?:any|baselayout|beep|call|context|deadkey|dk|if|index|layer|notany|nul|outs|platform|reset|return|save|set|store|use)\b/i,
+		alias: 'keyword'
+	},
+	'structural-keyword': {
+		pattern: /\b(?:ansi|begin|group|match|nomatch|unicode|using keys)\b/i,
+		alias: 'keyword'
+	},
+
+	'compile-target': {
+		pattern: /\$(?:keyman|keymanonly|keymanweb|kmfl|weaver):/i,
+		alias: 'property'
+	},
+
+	// U+####, x###, d### characters and numbers
+	'number': /\b(?:U\+[\dA-F]+|d\d+|x[\da-f]+|\d+)\b/i,
+	'operator': /[+>\\$]|\.\./,
+	'punctuation': /[()=,]/
 };

tests/languages/keyman/compile-target_feature.test

@@ -0,0 +1,27 @@
+$keyman:
+$kmfl:
+$weaver:
+$keymanweb:
+$keymanonly:
+
+c Note: return statement is only supported in .kmx in Keyman 13
+$keymanonly: + 'a' > 'a' return
+
+----------------------------------------------------
+
+[
+	["compile-target", "$keyman:"],
+	["compile-target", "$kmfl:"],
+	["compile-target", "$weaver:"],
+	["compile-target", "$keymanweb:"],
+	["compile-target", "$keymanonly:"],
+
+	["comment", "c Note: return statement is only supported in .kmx in Keyman 13"],
+
+	["compile-target", "$keymanonly:"],
+	["operator", "+"],
+	["string", "'a'"],
+	["operator", ">"],
+	["string", "'a'"],
+	["rule-keyword", "return"]
+]

tests/languages/keyman/header-keyword.test

@@ -0,0 +1,59 @@
+&baselayout
+&bitmap
+&capsononly
+&capsalwaysoff
+&shiftfreescaps
+&copyright
+&ethnologuecode
+&hotkey
+&includecodes
+&keyboardversion
+&kmw_embedcss
+&kmw_embedjs
+&kmw_helpfile
+&kmw_helptext
+&kmw_rtl
+&language
+&layer
+&layoutfile
+&message
+&mnemoniclayout
+&name
+&oldcharposmatching
+&platform
+&targets
+&version
+&visualkeyboard
+&windowslanguages
+
+----------------------------------------------------
+
+[
+	["header-keyword", "&baselayout"],
+	["header-keyword", "&bitmap"],
+	["header-keyword", "&capsononly"],
+	["header-keyword", "&capsalwaysoff"],
+	["header-keyword", "&shiftfreescaps"],
+	["header-keyword", "&copyright"],
+	["header-keyword", "&ethnologuecode"],
+	["header-keyword", "&hotkey"],
+	["header-keyword", "&includecodes"],
+	["header-keyword", "&keyboardversion"],
+	["header-keyword", "&kmw_embedcss"],
+	["header-keyword", "&kmw_embedjs"],
+	["header-keyword", "&kmw_helpfile"],
+	["header-keyword", "&kmw_helptext"],
+	["header-keyword", "&kmw_rtl"],
+	["header-keyword", "&language"],
+	["header-keyword", "&layer"],
+	["header-keyword", "&layoutfile"],
+	["header-keyword", "&message"],
+	["header-keyword", "&mnemoniclayout"],
+	["header-keyword", "&name"],
+	["header-keyword", "&oldcharposmatching"],
+	["header-keyword", "&platform"],
+	["header-keyword", "&targets"],
+	["header-keyword", "&version"],
+	["header-keyword", "&visualkeyboard"],
+	["header-keyword", "&windowslanguages"]
+]

tests/languages/keyman/header-statement.test

@@ -0,0 +1,29 @@
+bitmap
+bitmaps
+caps on only
+caps always off
+shift frees caps
+copyright
+hotkey
+language
+layout
+message
+name
+version
+
+----------------------------------------------------
+
+[
+	["header-statement", "bitmap"],
+	["header-statement", "bitmaps"],
+	["header-statement", "caps on only"],
+	["header-statement", "caps always off"],
+	["header-statement", "shift frees caps"],
+	["header-statement", "copyright"],
+	["header-statement", "hotkey"],
+	["header-statement", "language"],
+	["header-statement", "layout"],
+	["header-statement", "message"],
+	["header-statement", "name"],
+	["header-statement", "version"]
+]

tests/languages/keyman/operator_feature.test

@@ -1,15 +1,17 @@
-+ >
-\ ,
-( )
++ > \
+
+$vowel_a
+'a' .. 'z'
 
 ----------------------------------------------------
 
 [
-	["operator", "+"], ["operator", ">"],
-	["operator", "\\"], ["operator", ","],
-	["operator", "("], ["operator", ")"]
+	["operator", "+"], ["operator", ">"], ["operator", "\\"],
+
+	["operator", "$"], "vowel_a\r\n",
+	["string", "'a'"], ["operator", ".."], ["string", "'z'"]
 ]
 
 ----------------------------------------------------
 
-Checks for all operators.
+Checks for all operators.

tests/languages/keyman/punctuation_feature.test

@@ -0,0 +1,9 @@
+( )
+, =
+
+----------------------------------------------------
+
+[
+	["punctuation", "("], ["punctuation", ")"],
+	["punctuation", ","], ["punctuation", "="]
+]