Julia: Improved strings, comments, and other patterns (#2363)

components/prism-julia.js

@@ -1,13 +1,31 @@
-Prism.languages.julia= {
+Prism.languages.julia = {
 	'comment': {
-		pattern: /(^|[^\\])#.*/,
+		// support one level of nested comments
+		// https://github.com/JuliaLang/julia/pull/6128
+		pattern: /(^|[^\\])(?:#=(?:[^#=]|=(?!#)|#(?!=)|#=(?:[^#=]|=(?!#)|#(?!=))*=#)*=#|#.*)/,
 		lookbehind: true
 	},
-	'string': /("""|''')[\s\S]+?\1|("|')(?:\\.|(?!\2)[^\\\r\n])*\2/,
-	'keyword' : /\b(?:abstract|baremodule|begin|bitstype|break|catch|ccall|const|continue|do|else|elseif|end|export|finally|for|function|global|if|immutable|import|importall|in|let|local|macro|module|print|println|quote|return|struct|try|type|typealias|using|while)\b/,
-	'boolean' : /\b(?:true|false)\b/,
-	'number' : /(?:\b(?=\d)|\B(?=\.))(?:0[box])?(?:[\da-f]+\.?\d*|\.\d+)(?:[efp][+-]?\d+)?j?/i,
-	'operator': /[-+*^%÷&$\\]=?|\/[\/=]?|!=?=?|\|[=>]?|<(?:<=?|[=:])?|>(?:=|>>?=?)?|==?=?|[~≠≤≥]/,
-	'punctuation' : /[{}[\];(),.:]/,
-	'constant': /\b(?:(?:NaN|Inf)(?:16|32|64)?)\b/
+	'regex': {
+		// https://docs.julialang.org/en/v1/manual/strings/#Regular-Expressions-1
+		pattern: /r"(?:\\.|[^"\\\r\n])*"[imsx]{0,4}/,
+		greedy: true
+	},
+	'string': {
+		// https://docs.julialang.org/en/v1/manual/strings/#man-characters-1
+		// https://docs.julialang.org/en/v1/manual/strings/#String-Basics-1
+		// https://docs.julialang.org/en/v1/manual/strings/#non-standard-string-literals-1
+		// https://docs.julialang.org/en/v1/manual/running-external-programs/#Running-External-Programs-1
+		pattern: /"""[\s\S]+?"""|\w*"(?:\\.|[^"\\\r\n])*"|(^|[^\w'])'(?:\\[^\r\n][^'\r\n]*|[^\\\r\n])'|`(?:[^\\`\r\n]|\\.)*`/,
+		lookbehind: true,
+		greedy: true
+	},
+	'keyword': /\b(?:abstract|baremodule|begin|bitstype|break|catch|ccall|const|continue|do|else|elseif|end|export|finally|for|function|global|if|immutable|import|importall|in|let|local|macro|module|print|println|quote|return|struct|try|type|typealias|using|while)\b/,
+	'boolean': /\b(?:true|false)\b/,
+	'number': /(?:\b(?=\d)|\B(?=\.))(?:0[box])?(?:[\da-f]+(?:_[\da-f]+)*\.?(?:\d+(?:_\d+)*)?|\.\d+(?:_\d+)*)(?:[efp][+-]?\d+(?:_\d+)*)?j?/i,
+	// https://docs.julialang.org/en/v1/manual/mathematical-operations/
+	// https://docs.julialang.org/en/v1/manual/mathematical-operations/#Operator-Precedence-and-Associativity-1
+	'operator': /&&|\|\||[-+*^%÷⊻&$\\]=?|\/[\/=]?|!=?=?|\|[=>]?|<(?:<=?|[=:|])?|>(?:=|>>?=?)?|==?=?|[~≠≤≥'√∛]/,
+	'punctuation': /::?|[{}[\]();,.?]/,
+	// https://docs.julialang.org/en/v1/base/numbers/#Base.im
+	'constant': /\b(?:(?:NaN|Inf)(?:16|32|64)?|im|pi|e|catalan|eulergamma|golden)\b|[πℯγφ]/
 };

tests/languages/julia/comment_feature.test

@@ -1,13 +1,22 @@
 #
 # foobar
+#=#
+#=
+ multi line
+  =#
+
+#= #= nested =# =#
 
 ----------------------------------------------------
 
 [
 	["comment", "#"],
-	["comment", "# foobar"]
+	["comment", "# foobar"],
+	["comment", "#=#"],
+	["comment", "#=\r\n multi line\r\n  =#"],
+	["comment", "#= #= nested =# =#"]
 ]
 
 ----------------------------------------------------
 
-Checks for comments.
+Checks for comments.

tests/languages/julia/constant_feature.test

@@ -1,6 +1,13 @@
 NaN NaN16 NaN32 NaN64
 Inf Inf16 Inf32 Inf64
 
+im
+pi π
+e ℯ
+catalan
+eulergamma γ
+golden φ
+
 ----------------------------------------------------
 
 [
@@ -11,9 +18,20 @@ Inf Inf16 Inf32 Inf64
 	["constant", "Inf"],
 	["constant", "Inf16"],
 	["constant", "Inf32"],
-	["constant", "Inf64"]
+	["constant", "Inf64"],
+
+	["constant", "im"],
+	["constant", "pi"],
+	["constant", "π"],
+	["constant", "e"],
+	["constant", "ℯ"],
+	["constant", "catalan"],
+	["constant", "eulergamma"],
+	["constant", "γ"],
+	["constant", "golden"],
+	["constant", "φ"]
 ]
 
 ----------------------------------------------------
 
-Checks for all constants.
+Checks for all constants.

tests/languages/julia/issue2360.test

@@ -0,0 +1,39 @@
+"""
+	count_consecutive(str::AbstractString, start::Int)
+
+Counts the number of identical characters in a given string `str` starting from index `start`.
+
+# Examples
+
+```julia-repl
+julia> count_consecutive("AAAB", 3)
+('A', 3)
+
+julia> count_consecutive("x y z", 3)
+('y', 1)
+```
+"""
+function count_consecutive(str, start)
+	...
+end
+
+----------------------------------------------------
+
+[
+	["string", "\"\"\"\r\n\tcount_consecutive(str::AbstractString, start::Int)\r\n\r\nCounts the number of identical characters in a given string `str` starting from index `start`.\r\n\r\n# Examples\r\n\r\n```julia-repl\r\njulia> count_consecutive(\"AAAB\", 3)\r\n('A', 3)\r\n\r\njulia> count_consecutive(\"x y z\", 3)\r\n('y', 1)\r\n```\r\n\"\"\""],
+	["keyword", "function"],
+	" count_consecutive",
+	["punctuation", "("],
+	"str",
+	["punctuation", ","],
+	" start",
+	["punctuation", ")"],
+	["punctuation", "."],
+	["punctuation", "."],
+	["punctuation", "."],
+	["keyword", "end"]
+]
+
+----------------------------------------------------
+
+Multi-line strings didn't work correctly.

tests/languages/julia/number_feature.test

@@ -1,7 +1,10 @@
 0b0011
 0o274
 0xBadFace
+0x123456789abcdef
 42
+1.
+.5
 1.23
 1e10
 2.5e-4
@@ -10,24 +13,35 @@
 0x1p0
 0x1.8p3
 0x.4p-1
+10_000
+0.000_000_005
+0xdead_beef
+0b1011_0010
 
 ----------------------------------------------------
 
 [
 	["number", "0b0011"],
 	["number", "0o274"],
 	["number", "0xBadFace"],
+	["number", "0x123456789abcdef"],
 	["number", "42"],
+	["number", "1."],
+	["number", ".5"],
 	["number", "1.23"],
 	["number", "1e10"],
 	["number", "2.5e-4"],
 	["number", "0.5f0"],
 	["number", "2.5f-4"],
 	["number", "0x1p0"],
 	["number", "0x1.8p3"],
-	["number", "0x.4p-1"]
+	["number", "0x.4p-1"],
+	["number", "10_000"],
+	["number", "0.000_000_005"],
+	["number", "0xdead_beef"],
+	["number", "0b1011_0010"]
 ]
 
 ----------------------------------------------------
 
-Checks for binary, octal, hexadecimal and decimal numbers.
+Checks for binary, octal, hexadecimal and decimal numbers.

tests/languages/julia/operator_feature.test

@@ -7,13 +7,18 @@
 % %=
 ÷ ÷=
 ! != !==
-& &=
-| |= |>
+& &= &&
+⊻ ⊻=
+| |= ||
+|> <|
 $ $=
 < <= <: << <<=
 > >= >> >>= >>> >>>=
 = == ===
-~ ≠ ≤ ≥
+~ ≠ ≤ ≥ √ ∛
+
+A'b
+A'b''
 
 ----------------------------------------------------
 
@@ -27,15 +32,24 @@ $ $=
 	["operator", "%"], ["operator", "%="],
 	["operator", "÷"], ["operator", "÷="],
 	["operator", "!"], ["operator", "!="], ["operator", "!=="],
-	["operator", "&"], ["operator", "&="],
-	["operator", "|"], ["operator", "|="], ["operator", "|>"],
+	["operator", "&"], ["operator", "&="], ["operator", "&&"],
+	["operator", "⊻"], ["operator", "⊻="],
+	["operator", "|"], ["operator", "|="], ["operator", "||"],
+	["operator", "|>"], ["operator", "<|"],
 	["operator", "$"], ["operator", "$="],
 	["operator", "<"], ["operator", "<="], ["operator", "<:"], ["operator", "<<"], ["operator", "<<="],
 	["operator", ">"], ["operator", ">="], ["operator", ">>"], ["operator", ">>="], ["operator", ">>>"], ["operator", ">>>="],
 	["operator", "="], ["operator", "=="], ["operator", "==="],
-	["operator", "~"], ["operator", "≠"], ["operator", "≤"], ["operator", "≥"]
+	["operator", "~"], ["operator", "≠"], ["operator", "≤"], ["operator", "≥"], ["operator", "√"], ["operator", "∛"],
+	"\r\n\r\nA",
+	["operator", "'"],
+	"b\r\nA",
+	["operator", "'"],
+	"b",
+	["operator", "'"],
+	["operator", "'"]
 ]
 
 ----------------------------------------------------
 
-Checks for operators.
+Checks for operators.

tests/languages/julia/punctuation_feature.test

@@ -0,0 +1,26 @@
+{ } [ ] ( )
+; ,
+.
+::
+? :
+
+----------------------------------------------------
+
+[
+	["punctuation", "{"],
+	["punctuation", "}"],
+	["punctuation", "["],
+	["punctuation", "]"],
+	["punctuation", "("],
+	["punctuation", ")"],
+	["punctuation", ";"],
+	["punctuation", ","],
+	["punctuation", "."],
+	["punctuation", "::"],
+	["punctuation", "?"],
+	["punctuation", ":"]
+]
+
+----------------------------------------------------
+
+Checks for punctuation.

tests/languages/julia/regex_feature.test

@@ -0,0 +1,17 @@
+r"foo"i
+r"(a|b)(c)?(d)"
+r"^\s*(?:#\s*(.*?)\s*$|$)"
+r"a+.*b+.*?d$"ism
+
+----------------------------------------------------
+
+[
+	["regex", "r\"foo\"i"],
+	["regex", "r\"(a|b)(c)?(d)\""],
+	["regex", "r\"^\\s*(?:#\\s*(.*?)\\s*$|$)\""],
+	["regex", "r\"a+.*b+.*?d$\"ism"]
+]
+
+----------------------------------------------------
+
+Checks for regular expressions.

tests/languages/julia/string_feature.test

@@ -1,29 +1,67 @@
 ""
 "fo\"o"
+"\xe2\x88"
+
 'x'
 '\''
 '\u2200'
+'\x80'
+'\xe2\x88'
+'∀'
+
 """foo"""
 """fo"o
 bar"""
-'''foo'''
-'''fo'o
-bar'''
+
+`echo hello`
+`echo "foo bar"`
+
+# non-standard string
+s"\g<0>1"
+b"DATA\xff\u2200"
+v"0.3-"
+raw"\\ \\\""
+
+# not a character
+A'b
+A'b''
 
 ----------------------------------------------------
 
 [
 	["string", "\"\""],
 	["string", "\"fo\\\"o\""],
+	["string", "\"\\xe2\\x88\""],
+
 	["string", "'x'"],
 	["string", "'\\''"],
 	["string", "'\\u2200'"],
+	["string", "'\\x80'"],
+	["string", "'\\xe2\\x88'"],
+	["string", "'∀'"],
+
 	["string", "\"\"\"foo\"\"\""],
 	["string", "\"\"\"fo\"o\r\nbar\"\"\""],
-	["string", "'''foo'''"],
-	["string", "'''fo'o\r\nbar'''"]
+
+	["string", "`echo hello`"],
+	["string", "`echo \"foo bar\"`"],
+
+	["comment", "# non-standard string"],
+	["string", "s\"\\g<0>1\""],
+	["string", "b\"DATA\\xff\\u2200\""],
+	["string", "v\"0.3-\""],
+	["string", "raw\"\\\\ \\\\\\\"\""],
+
+	["comment", "# not a character"],
+	"\r\nA",
+	["operator", "'"],
+	"b\r\nA",
+	["operator", "'"],
+	"b",
+	["operator", "'"],
+	["operator", "'"]
 ]
 
 ----------------------------------------------------
 
-Checks for strings and characters.
+Checks for strings and characters.