Go: Added char token and improved string and number tokens (#3208)

components/prism-go.js

@@ -1,12 +1,28 @@
 Prism.languages.go = Prism.languages.extend('clike', {
 	'string': {
-		pattern: /(["'`])(?:\\[\s\S]|(?!\1)[^\\])*\1/,
+		pattern: /(^|[^\\])"(?:\\.|[^"\\\r\n])*"|`[^`]*`/,
+		lookbehind: true,
 		greedy: true
 	},
 	'keyword': /\b(?:break|case|chan|const|continue|default|defer|else|fallthrough|for|func|go(?:to)?|if|import|interface|map|package|range|return|select|struct|switch|type|var)\b/,
 	'boolean': /\b(?:_|false|iota|nil|true)\b/,
-	'number': /(?:\b0x[a-f\d]+|(?:\b\d+(?:\.\d*)?|\B\.\d+)(?:e[-+]?\d+)?)i?/i,
+	'number': [
+		// binary and octal integers
+		/\b0(?:b[01_]+|o[0-7_]+)i?\b/i,
+		// hexadecimal integers and floats
+		/\b0x(?:[a-f\d_]+(?:\.[a-f\d_]*)?|\.[a-f\d_]+)(?:p[+-]?\d+(?:_\d+)*)?i?(?!\w)/i,
+		// decimal integers and floats
+		/(?:\b\d[\d_]*(?:\.[\d_]*)?|\B\.\d[\d_]*)(?:e[+-]?[\d_]+)?i?(?!\w)/i
+	],
 	'operator': /[*\/%^!=]=?|\+[=+]?|-[=-]?|\|[=|]?|&(?:=|&|\^=?)?|>(?:>=?|=)?|<(?:<=?|=|-)?|:=|\.\.\./,
 	'builtin': /\b(?:append|bool|byte|cap|close|complex|complex(?:64|128)|copy|delete|error|float(?:32|64)|u?int(?:8|16|32|64)?|imag|len|make|new|panic|print(?:ln)?|real|recover|rune|string|uintptr)\b/
 });
+
+Prism.languages.insertBefore('go', 'string', {
+	'char': {
+		pattern: /'(?:\\.|[^'\\\r\n]){0,10}'/,
+		greedy: true
+	}
+});
+
 delete Prism.languages.go['class-name'];

tests/languages/go/char_feature.test

@@ -0,0 +1,29 @@
+'a'
+'ä'
+'本'
+'\t'
+'\000'
+'\007'
+'\377'
+'\x07'
+'\xff'
+'\u12e4'
+'\U00101234'
+'\''
+
+----------------------------------------------------
+
+[
+	["char", "'a'"],
+	["char", "'ä'"],
+	["char", "'本'"],
+	["char", "'\\t'"],
+	["char", "'\\000'"],
+	["char", "'\\007'"],
+	["char", "'\\377'"],
+	["char", "'\\x07'"],
+	["char", "'\\xff'"],
+	["char", "'\\u12e4'"],
+	["char", "'\\U00101234'"],
+	["char", "'\\''"]
+]

tests/languages/go/number_feature.test

@@ -2,19 +2,48 @@
 0600
 0xBadFace
 170141183460469231731687303715884105727
+42
+4_2
+0600
+0_600
+0o600
+0O600       // second character is capital letter 'O'
+0xBadFace
+0xBad_Face
+0x_67_7a_2f_cc_40_c6
+170141183460469231731687303715884105727
+170_141183_460469_231731_687303_715884_105727
+
+0.
 72.40
-072.40
+072.40       // == 72.40
 2.71828
 1.e+0
 6.67428e-11
 1E6
+.25
+.12345E+5
+1_5.         // == 15.0
+0.15e+0_2    // == 15.0
+
+0x1p-2       // == 0.25
+0x2.p10      // == 2048.0
+0x1.Fp+0     // == 1.9375
+0X.8p-0      // == 0.5
+0X_1FFFP-16  // == 0.1249847412109375
+
 0i
-011i
+0123i         // == 123i for backward-compatibility
+0o123i        // == 0o123 * 1i == 83i
+0xabci        // == 0xabc * 1i == 2748i
 0.i
 2.71828i
 1.e+0i
 6.67428e-11i
 1E6i
+.25i
+.12345E+5i
+0x1p-2i       // == 0x1p-2 * 1i == 0.25i
 
 ----------------------------------------------------
 
@@ -23,21 +52,51 @@
 	["number", "0600"],
 	["number", "0xBadFace"],
 	["number", "170141183460469231731687303715884105727"],
+	["number", "42"],
+	["number", "4_2"],
+	["number", "0600"],
+	["number", "0_600"],
+	["number", "0o600"],
+	["number", "0O600"],
+	["comment", "// second character is capital letter 'O'"],
+	["number", "0xBadFace"],
+	["number", "0xBad_Face"],
+	["number", "0x_67_7a_2f_cc_40_c6"],
+	["number", "170141183460469231731687303715884105727"],
+	["number", "170_141183_460469_231731_687303_715884_105727"],
+
+	["number", "0."],
 	["number", "72.40"],
-	["number", "072.40"],
+	["number", "072.40"], ["comment", "// == 72.40"],
 	["number", "2.71828"],
 	["number", "1.e+0"],
 	["number", "6.67428e-11"],
 	["number", "1E6"],
+	["number", ".25"],
+	["number", ".12345E+5"],
+	["number", "1_5."], ["comment", "// == 15.0"],
+	["number", "0.15e+0_2"], ["comment", "// == 15.0"],
+
+	["number", "0x1p-2"], ["comment", "// == 0.25"],
+	["number", "0x2.p10"], ["comment", "// == 2048.0"],
+	["number", "0x1.Fp+0"], ["comment", "// == 1.9375"],
+	["number", "0X.8p-0"], ["comment", "// == 0.5"],
+	["number", "0X_1FFFP-16"], ["comment", "// == 0.1249847412109375"],
+
 	["number", "0i"],
-	["number", "011i"],
+	["number", "0123i"], ["comment", "// == 123i for backward-compatibility"],
+	["number", "0o123i"], ["comment", "// == 0o123 * 1i == 83i"],
+	["number", "0xabci"], ["comment", "// == 0xabc * 1i == 2748i"],
 	["number", "0.i"],
 	["number", "2.71828i"],
 	["number", "1.e+0i"],
 	["number", "6.67428e-11i"],
-	["number", "1E6i"]
+	["number", "1E6i"],
+	["number", ".25i"],
+	["number", ".12345E+5i"],
+	["number", "0x1p-2i"], ["comment", "// == 0x1p-2 * 1i == 0.25i"]
 ]
 
 ----------------------------------------------------
 
-Checks for integers, floats and imaginary numbers.
+Checks for integers, floats and imaginary numbers.

tests/languages/go/punctuation_feature.test

@@ -0,0 +1,18 @@
+( ) { } [ ]
+, ; . :
+
+----------------------------------------------------
+
+[
+	["punctuation", "("],
+	["punctuation", ")"],
+	["punctuation", "{"],
+	["punctuation", "}"],
+	["punctuation", "["],
+	["punctuation", "]"],
+
+	["punctuation", ","],
+	["punctuation", ";"],
+	["punctuation", "."],
+	["punctuation", ":"]
+]

tests/languages/go/string_feature.test

@@ -1,13 +1,10 @@
-'a'
-'ä'
-'本'
-'\t'
-'\xff'
-'\u12e4'
-
+``
 `abc`
 `\n
 \n`
+`\`
+
+""
 "\n"
 "\""
 "Hello, world!\n"
@@ -17,14 +14,12 @@
 ----------------------------------------------------
 
 [
-	["string", "'a'"],
-	["string", "'ä'"],
-	["string", "'本'"],
-	["string", "'\\t'"],
-	["string", "'\\xff'"],
-	["string", "'\\u12e4'"],
+	["string", "``"],
 	["string", "`abc`"],
 	["string", "`\\n\r\n\\n`"],
+	["string", "`\\`"],
+
+	["string", "\"\""],
 	["string", "\"\\n\""],
 	["string", "\"\\\"\""],
 	["string", "\"Hello, world!\\n\""],
@@ -34,4 +29,4 @@
 
 ----------------------------------------------------
 
-Checks for runes and strings.
+Checks for runes and strings.