Skip to content

Commit 810464a

Browse files
otbutzotbutz
authoredOct 26, 2023
Update Java lexer (#873)
·
v2.15.0v2.10.0
1 parent 77e9146 commit 810464a

File tree

5 files changed

+284
-180
lines changed

5 files changed

+284
-180
lines changed
 

‎lexers/embedded/java.xml

+130-58
Original file line numberDiff line numberDiff line change
@@ -5,116 +5,188 @@
55
<filename>*.java</filename>
66
<mime_type>text/x-java</mime_type>
77
<dot_all>true</dot_all>
8-
<ensure_nl>true</ensure_nl>
98
</config>
109
<rules>
11-
<state name="class">
12-
<rule pattern="([^\W\d]|\$)[\w$]*">
13-
<token type="NameClass"/>
14-
<pop depth="1"/>
15-
</rule>
16-
</state>
17-
<state name="import">
18-
<rule pattern="[\w.]+\*?">
19-
<token type="NameNamespace"/>
20-
<pop depth="1"/>
21-
</rule>
22-
</state>
2310
<state name="root">
11+
<rule pattern="(^\s*)((?:(?:public|private|protected|static|strictfp)(?:\s+))*)(record)\b">
12+
<bygroups>
13+
<token type="TextWhitespace" />
14+
<usingself state="root" />
15+
<token type="KeywordDeclaration" />
16+
</bygroups>
17+
<push state="class" />
18+
</rule>
2419
<rule pattern="[^\S\n]+">
25-
<token type="Text"/>
20+
<token type="TextWhitespace" />
2621
</rule>
27-
<rule pattern="//.*?\n">
28-
<token type="CommentSingle"/>
22+
<rule pattern="(//.*?)(\n)">
23+
<bygroups>
24+
<token type="CommentSingle" />
25+
<token type="TextWhitespace" />
26+
</bygroups>
2927
</rule>
3028
<rule pattern="/\*.*?\*/">
31-
<token type="CommentMultiline"/>
29+
<token type="CommentMultiline" />
3230
</rule>
33-
<rule pattern="(assert|break|case|catch|continue|default|do|else|finally|for|if|goto|instanceof|new|return|switch|this|throw|try|while)\b">
34-
<token type="Keyword"/>
31+
<rule
32+
pattern="(assert|break|case|catch|continue|default|do|else|finally|for|if|goto|instanceof|new|return|switch|this|throw|try|while)\b">
33+
<token type="Keyword" />
3534
</rule>
3635
<rule pattern="((?:(?:[^\W\d]|\$)[\w.\[\]$&lt;&gt;]*\s+)+?)((?:[^\W\d]|\$)[\w$]*)(\s*)(\()">
3736
<bygroups>
38-
<usingself state="root"/>
39-
<token type="NameFunction"/>
40-
<token type="Text"/>
41-
<token type="Operator"/>
37+
<usingself state="root" />
38+
<token type="NameFunction" />
39+
<token type="TextWhitespace" />
40+
<token type="Punctuation" />
4241
</bygroups>
4342
</rule>
4443
<rule pattern="@[^\W\d][\w.]*">
45-
<token type="NameDecorator"/>
44+
<token type="NameDecorator" />
4645
</rule>
47-
<rule pattern="(abstract|const|enum|extends|final|implements|native|private|protected|public|static|strictfp|super|synchronized|throws|transient|volatile)\b">
48-
<token type="KeywordDeclaration"/>
46+
<rule
47+
pattern="(abstract|const|enum|extends|final|implements|native|private|protected|public|sealed|static|strictfp|super|synchronized|throws|transient|volatile|yield)\b">
48+
<token type="KeywordDeclaration" />
4949
</rule>
5050
<rule pattern="(boolean|byte|char|double|float|int|long|short|void)\b">
51-
<token type="KeywordType"/>
51+
<token type="KeywordType" />
5252
</rule>
5353
<rule pattern="(package)(\s+)">
5454
<bygroups>
55-
<token type="KeywordNamespace"/>
56-
<token type="Text"/>
55+
<token type="KeywordNamespace" />
56+
<token type="TextWhitespace" />
5757
</bygroups>
58-
<push state="import"/>
58+
<push state="import" />
5959
</rule>
6060
<rule pattern="(true|false|null)\b">
61-
<token type="KeywordConstant"/>
61+
<token type="KeywordConstant" />
62+
</rule>
63+
<rule pattern="(class|interface)\b">
64+
<token type="KeywordDeclaration" />
65+
<push state="class" />
6266
</rule>
63-
<rule pattern="(class|interface)(\s+)">
67+
<rule pattern="(var)(\s+)">
6468
<bygroups>
65-
<token type="KeywordDeclaration"/>
66-
<token type="Text"/>
69+
<token type="KeywordDeclaration" />
70+
<token type="TextWhitespace" />
6771
</bygroups>
68-
<push state="class"/>
72+
<push state="var" />
6973
</rule>
7074
<rule pattern="(import(?:\s+static)?)(\s+)">
7175
<bygroups>
72-
<token type="KeywordNamespace"/>
73-
<token type="Text"/>
76+
<token type="KeywordNamespace" />
77+
<token type="TextWhitespace" />
7478
</bygroups>
75-
<push state="import"/>
79+
<push state="import" />
7680
</rule>
77-
<rule pattern="&#34;(\\\\|\\&#34;|[^&#34;])*&#34;">
78-
<token type="LiteralString"/>
81+
<rule pattern="&quot;&quot;&quot;\n">
82+
<token type="LiteralString" />
83+
<push state="multiline_string" />
7984
</rule>
80-
<rule pattern="&#39;\\.&#39;|&#39;[^\\]&#39;|&#39;\\u[0-9a-fA-F]{4}&#39;">
81-
<token type="LiteralStringChar"/>
85+
<rule pattern="&quot;">
86+
<token type="LiteralString" />
87+
<push state="string" />
8288
</rule>
83-
<rule pattern="\d+[LlUu]*">
84-
<token type="LiteralNumberInteger"/>
89+
<rule pattern="&#x27;\\.&#x27;|&#x27;[^\\]&#x27;|&#x27;\\u[0-9a-fA-F]{4}&#x27;">
90+
<token type="LiteralStringChar" />
8591
</rule>
8692
<rule pattern="(\.)((?:[^\W\d]|\$)[\w$]*)">
8793
<bygroups>
88-
<token type="Operator"/>
89-
<token type="NameAttribute"/>
94+
<token type="Punctuation" />
95+
<token type="NameAttribute" />
96+
</bygroups>
97+
</rule>
98+
<rule pattern="^(\s*)(default)(:)">
99+
<bygroups>
100+
<token type="TextWhitespace" />
101+
<token type="Keyword" />
102+
<token type="Punctuation" />
90103
</bygroups>
91104
</rule>
92-
<rule pattern="^\s*([^\W\d]|\$)[\w$]*:">
93-
<token type="NameLabel"/>
105+
<rule pattern="^(\s*)((?:[^\W\d]|\$)[\w$]*)(:)">
106+
<bygroups>
107+
<token type="TextWhitespace" />
108+
<token type="NameLabel" />
109+
<token type="Punctuation" />
110+
</bygroups>
94111
</rule>
95112
<rule pattern="([^\W\d]|\$)[\w$]*">
96-
<token type="Name"/>
113+
<token type="Name" />
97114
</rule>
98-
<rule pattern="([0-9][0-9_]*\.([0-9][0-9_]*)?|\.[0-9][0-9_]*)([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)[pP][+\-]?[0-9][0-9_]*[fFdD]?">
99-
<token type="LiteralNumberFloat"/>
115+
<rule
116+
pattern="([0-9][0-9_]*\.([0-9][0-9_]*)?|\.[0-9][0-9_]*)([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)[pP][+\-]?[0-9][0-9_]*[fFdD]?">
117+
<token type="LiteralNumberFloat" />
100118
</rule>
101119
<rule pattern="0[xX][0-9a-fA-F][0-9a-fA-F_]*[lL]?">
102-
<token type="LiteralNumberHex"/>
120+
<token type="LiteralNumberHex" />
103121
</rule>
104122
<rule pattern="0[bB][01][01_]*[lL]?">
105-
<token type="LiteralNumberBin"/>
123+
<token type="LiteralNumberBin" />
106124
</rule>
107125
<rule pattern="0[0-7_]+[lL]?">
108-
<token type="LiteralNumberOct"/>
126+
<token type="LiteralNumberOct" />
109127
</rule>
110128
<rule pattern="0|[1-9][0-9_]*[lL]?">
111-
<token type="LiteralNumberInteger"/>
129+
<token type="LiteralNumberInteger" />
112130
</rule>
113-
<rule pattern="[~^*!%&amp;\[\](){}&lt;&gt;|+=:;,./?-]">
114-
<token type="Operator"/>
131+
<rule pattern="[~^*!%&amp;\[\]&lt;&gt;|+=/?-]">
132+
<token type="Operator" />
133+
</rule>
134+
<rule pattern="[{}();:.,]">
135+
<token type="Punctuation" />
115136
</rule>
116137
<rule pattern="\n">
117-
<token type="Text"/>
138+
<token type="TextWhitespace" />
139+
</rule>
140+
</state>
141+
<state name="class">
142+
<rule pattern="\s+">
143+
<token type="Text" />
144+
</rule>
145+
<rule pattern="([^\W\d]|\$)[\w$]*">
146+
<token type="NameClass" />
147+
<pop depth="1" />
148+
</rule>
149+
</state>
150+
<state name="var">
151+
<rule pattern="([^\W\d]|\$)[\w$]*">
152+
<token type="Name" />
153+
<pop depth="1" />
154+
</rule>
155+
</state>
156+
<state name="import">
157+
<rule pattern="[\w.]+\*?">
158+
<token type="NameNamespace" />
159+
<pop depth="1" />
160+
</rule>
161+
</state>
162+
<state name="multiline_string">
163+
<rule pattern="&quot;&quot;&quot;">
164+
<token type="LiteralString" />
165+
<pop depth="1" />
166+
</rule>
167+
<rule pattern="&quot;">
168+
<token type="LiteralString" />
169+
</rule>
170+
<rule>
171+
<include state="string" />
172+
</rule>
173+
</state>
174+
<state name="string">
175+
<rule pattern="[^\\&quot;]+">
176+
<token type="LiteralString" />
177+
</rule>
178+
<rule pattern="\\\\">
179+
<token type="LiteralString" />
180+
</rule>
181+
<rule pattern="\\&quot;">
182+
<token type="LiteralString" />
183+
</rule>
184+
<rule pattern="\\">
185+
<token type="LiteralString" />
186+
</rule>
187+
<rule pattern="&quot;">
188+
<token type="LiteralString" />
189+
<pop depth="1" />
118190
</rule>
119191
</state>
120192
</rules>

‎lexers/testdata/cql.expected

+7-7
Original file line numberDiff line numberDiff line change
@@ -1066,19 +1066,19 @@
10661066
{"type":"TextWhitespace","value":"\n"},
10671067
{"type":"LiteralStringHeredoc","value":"'"},
10681068
{"type":"Keyword","value":"return"},
1069-
{"type":"Text","value":" "},
1069+
{"type":"TextWhitespace","value":" "},
10701070
{"type":"Name","value":"Double"},
1071-
{"type":"Operator","value":"."},
1071+
{"type":"Punctuation","value":"."},
10721072
{"type":"NameAttribute","value":"valueOf"},
1073-
{"type":"Operator","value":"("},
1073+
{"type":"Punctuation","value":"("},
10741074
{"type":"Name","value":"Math"},
1075-
{"type":"Operator","value":"."},
1075+
{"type":"Punctuation","value":"."},
10761076
{"type":"NameAttribute","value":"log"},
1077-
{"type":"Operator","value":"("},
1077+
{"type":"Punctuation","value":"("},
10781078
{"type":"Name","value":"input"},
1079-
{"type":"Operator","value":"."},
1079+
{"type":"Punctuation","value":"."},
10801080
{"type":"NameAttribute","value":"doubleValue"},
1081-
{"type":"Operator","value":"()));"},
1081+
{"type":"Punctuation","value":"()));"},
10821082
{"type":"LiteralStringHeredoc","value":"'"},
10831083
{"type":"Punctuation","value":";"},
10841084
{"type":"TextWhitespace","value":"\n"}

‎lexers/testdata/java.actual

+9
Original file line numberDiff line numberDiff line change
@@ -23,4 +23,13 @@ final class TargetUnsafeRefArrayAccess {
2323
@Alias
2424
@RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayIndexShift, declClass = Object[].class)
2525
public static int REF_ELEMENT_SHIFT;
26+
27+
public static void test() {
28+
System.out.println("""
29+
Hello, world!
30+
This is a multi-line string!
31+
It can also contain "quotes" and 'apostrophes' without breaking.
32+
We only need to escape \""" inside it.
33+
""");
34+
}
2635
}

‎lexers/testdata/java.expected

+131-108
Original file line numberDiff line numberDiff line change
@@ -1,195 +1,218 @@
11
[
22
{"type":"KeywordNamespace","value":"package"},
3-
{"type":"Text","value":" "},
3+
{"type":"TextWhitespace","value":" "},
44
{"type":"NameNamespace","value":"example"},
5-
{"type":"Operator","value":";"},
6-
{"type":"Text","value":"\n\n"},
5+
{"type":"Punctuation","value":";"},
6+
{"type":"TextWhitespace","value":"\n\n"},
77
{"type":"KeywordNamespace","value":"import"},
8-
{"type":"Text","value":" "},
8+
{"type":"TextWhitespace","value":" "},
99
{"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Alias"},
10-
{"type":"Operator","value":";"},
11-
{"type":"Text","value":"\n"},
10+
{"type":"Punctuation","value":";"},
11+
{"type":"TextWhitespace","value":"\n"},
1212
{"type":"KeywordNamespace","value":"import"},
13-
{"type":"Text","value":" "},
13+
{"type":"TextWhitespace","value":" "},
1414
{"type":"NameNamespace","value":"com.oracle.svm.core.annotate.RecomputeFieldValue"},
15-
{"type":"Operator","value":";"},
16-
{"type":"Text","value":"\n"},
15+
{"type":"Punctuation","value":";"},
16+
{"type":"TextWhitespace","value":"\n"},
1717
{"type":"KeywordNamespace","value":"import"},
18-
{"type":"Text","value":" "},
18+
{"type":"TextWhitespace","value":" "},
1919
{"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Targetclass"},
20-
{"type":"Operator","value":";"},
21-
{"type":"Text","value":"\n\n"},
20+
{"type":"Punctuation","value":";"},
21+
{"type":"TextWhitespace","value":"\n\n"},
2222
{"type":"NameDecorator","value":"@TargetClass"},
23-
{"type":"Operator","value":"("},
23+
{"type":"Punctuation","value":"("},
2424
{"type":"Name","value":"className"},
25-
{"type":"Text","value":" "},
25+
{"type":"TextWhitespace","value":" "},
2626
{"type":"Operator","value":"="},
27-
{"type":"Text","value":" "},
27+
{"type":"TextWhitespace","value":" "},
2828
{"type":"LiteralString","value":"\"io.netty.util.internal.cleanerJava6\""},
29-
{"type":"Operator","value":")"},
30-
{"type":"Text","value":"\n"},
29+
{"type":"Punctuation","value":")"},
30+
{"type":"TextWhitespace","value":"\n"},
3131
{"type":"KeywordDeclaration","value":"final"},
32-
{"type":"Text","value":" "},
32+
{"type":"TextWhitespace","value":" "},
3333
{"type":"KeywordDeclaration","value":"class"},
3434
{"type":"Text","value":" "},
3535
{"type":"NameClass","value":"TargetCleanerJava6"},
36-
{"type":"Text","value":" "},
37-
{"type":"Operator","value":"{"},
38-
{"type":"Text","value":"\n "},
36+
{"type":"TextWhitespace","value":" "},
37+
{"type":"Punctuation","value":"{"},
38+
{"type":"TextWhitespace","value":"\n "},
3939
{"type":"NameDecorator","value":"@Alias"},
40-
{"type":"Text","value":"\n "},
40+
{"type":"TextWhitespace","value":"\n "},
4141
{"type":"NameDecorator","value":"@RecomputeFieldValue"},
42-
{"type":"Operator","value":"("},
42+
{"type":"Punctuation","value":"("},
4343
{"type":"Name","value":"kind"},
44-
{"type":"Text","value":" "},
44+
{"type":"TextWhitespace","value":" "},
4545
{"type":"Operator","value":"="},
46-
{"type":"Text","value":" "},
46+
{"type":"TextWhitespace","value":" "},
4747
{"type":"Name","value":"RecomputeFieldValue"},
48-
{"type":"Operator","value":"."},
48+
{"type":"Punctuation","value":"."},
4949
{"type":"NameAttribute","value":"Kind"},
50-
{"type":"Operator","value":"."},
50+
{"type":"Punctuation","value":"."},
5151
{"type":"NameAttribute","value":"Fieldoffset"},
52-
{"type":"Operator","value":","},
53-
{"type":"Text","value":" "},
52+
{"type":"Punctuation","value":","},
53+
{"type":"TextWhitespace","value":" "},
5454
{"type":"Name","value":"declClassName"},
55-
{"type":"Text","value":" "},
55+
{"type":"TextWhitespace","value":" "},
5656
{"type":"Operator","value":"="},
57-
{"type":"Text","value":" "},
57+
{"type":"TextWhitespace","value":" "},
5858
{"type":"LiteralString","value":"\"java.nio.DirectBy\""},
59-
{"type":"Operator","value":")"},
60-
{"type":"Text","value":"\n "},
59+
{"type":"Punctuation","value":")"},
60+
{"type":"TextWhitespace","value":"\n "},
6161
{"type":"KeywordDeclaration","value":"private"},
62-
{"type":"Text","value":" "},
62+
{"type":"TextWhitespace","value":" "},
6363
{"type":"KeywordDeclaration","value":"static"},
64-
{"type":"Text","value":" "},
64+
{"type":"TextWhitespace","value":" "},
6565
{"type":"KeywordType","value":"long"},
66-
{"type":"Text","value":" "},
66+
{"type":"TextWhitespace","value":" "},
6767
{"type":"Name","value":"CLEANER_FIELD_OFFSET"},
68-
{"type":"Operator","value":";"},
69-
{"type":"Text","value":"\n"},
70-
{"type":"Operator","value":"}"},
71-
{"type":"Text","value":"\n\n"},
68+
{"type":"Punctuation","value":";"},
69+
{"type":"TextWhitespace","value":"\n"},
70+
{"type":"Punctuation","value":"}"},
71+
{"type":"TextWhitespace","value":"\n\n"},
7272
{"type":"NameDecorator","value":"@TargetClass"},
73-
{"type":"Operator","value":"("},
73+
{"type":"Punctuation","value":"("},
7474
{"type":"Name","value":"className"},
75-
{"type":"Text","value":" "},
75+
{"type":"TextWhitespace","value":" "},
7676
{"type":"Operator","value":"="},
77-
{"type":"Text","value":" "},
77+
{"type":"TextWhitespace","value":" "},
7878
{"type":"LiteralString","value":"\"io.netty.util.internal.PlatformDependent0\""},
79-
{"type":"Operator","value":")"},
80-
{"type":"Text","value":"\n"},
79+
{"type":"Punctuation","value":")"},
80+
{"type":"TextWhitespace","value":"\n"},
8181
{"type":"KeywordDeclaration","value":"final"},
82-
{"type":"Text","value":" "},
82+
{"type":"TextWhitespace","value":" "},
8383
{"type":"KeywordDeclaration","value":"class"},
8484
{"type":"Text","value":" "},
8585
{"type":"NameClass","value":"TargetPlatformDependent0"},
86-
{"type":"Text","value":" "},
87-
{"type":"Operator","value":"{"},
88-
{"type":"Text","value":"\n "},
86+
{"type":"TextWhitespace","value":" "},
87+
{"type":"Punctuation","value":"{"},
88+
{"type":"TextWhitespace","value":"\n "},
8989
{"type":"NameDecorator","value":"@Alias"},
90-
{"type":"Text","value":"\n "},
90+
{"type":"TextWhitespace","value":"\n "},
9191
{"type":"NameDecorator","value":"@RecomputeFieldValue"},
92-
{"type":"Operator","value":"("},
92+
{"type":"Punctuation","value":"("},
9393
{"type":"Name","value":"kind"},
94-
{"type":"Text","value":" "},
94+
{"type":"TextWhitespace","value":" "},
9595
{"type":"Operator","value":"="},
96-
{"type":"Text","value":" "},
96+
{"type":"TextWhitespace","value":" "},
9797
{"type":"Name","value":"RecomputeFieldValue"},
98-
{"type":"Operator","value":"."},
98+
{"type":"Punctuation","value":"."},
9999
{"type":"NameAttribute","value":"Kind"},
100-
{"type":"Operator","value":"."},
100+
{"type":"Punctuation","value":"."},
101101
{"type":"NameAttribute","value":"Fieldoffset"},
102-
{"type":"Operator","value":","},
103-
{"type":"Text","value":" "},
102+
{"type":"Punctuation","value":","},
103+
{"type":"TextWhitespace","value":" "},
104104
{"type":"Name","value":"declClassName"},
105-
{"type":"Text","value":" "},
105+
{"type":"TextWhitespace","value":" "},
106106
{"type":"Operator","value":"="},
107-
{"type":"Text","value":" "},
107+
{"type":"TextWhitespace","value":" "},
108108
{"type":"LiteralString","value":"\"java.nio.Buffer\""},
109-
{"type":"Operator","value":","},
110-
{"type":"Text","value":" "},
109+
{"type":"Punctuation","value":","},
110+
{"type":"TextWhitespace","value":" "},
111111
{"type":"Name","value":"rest"},
112-
{"type":"Text","value":" "},
112+
{"type":"TextWhitespace","value":" "},
113113
{"type":"Operator","value":"="},
114-
{"type":"Text","value":" "},
114+
{"type":"TextWhitespace","value":" "},
115115
{"type":"LiteralString","value":"\"IDK\""},
116-
{"type":"Operator","value":")"},
117-
{"type":"Text","value":"\n "},
116+
{"type":"Punctuation","value":")"},
117+
{"type":"TextWhitespace","value":"\n "},
118118
{"type":"KeywordDeclaration","value":"private"},
119-
{"type":"Text","value":" "},
119+
{"type":"TextWhitespace","value":" "},
120120
{"type":"KeywordDeclaration","value":"static"},
121-
{"type":"Text","value":" "},
121+
{"type":"TextWhitespace","value":" "},
122122
{"type":"KeywordType","value":"long"},
123-
{"type":"Text","value":" "},
123+
{"type":"TextWhitespace","value":" "},
124124
{"type":"Name","value":"ADDRESS_FIELD_OFFSET"},
125-
{"type":"Operator","value":";"},
126-
{"type":"Text","value":"\n"},
127-
{"type":"Operator","value":"}"},
128-
{"type":"Text","value":"\n\n"},
125+
{"type":"Punctuation","value":";"},
126+
{"type":"TextWhitespace","value":"\n"},
127+
{"type":"Punctuation","value":"}"},
128+
{"type":"TextWhitespace","value":"\n\n"},
129129
{"type":"NameDecorator","value":"@TargetClass"},
130-
{"type":"Operator","value":"("},
130+
{"type":"Punctuation","value":"("},
131131
{"type":"Name","value":"io"},
132-
{"type":"Operator","value":"."},
132+
{"type":"Punctuation","value":"."},
133133
{"type":"NameAttribute","value":"netty"},
134-
{"type":"Operator","value":"."},
134+
{"type":"Punctuation","value":"."},
135135
{"type":"NameAttribute","value":"util"},
136-
{"type":"Operator","value":"."},
136+
{"type":"Punctuation","value":"."},
137137
{"type":"NameAttribute","value":"internal"},
138-
{"type":"Operator","value":"."},
138+
{"type":"Punctuation","value":"."},
139139
{"type":"NameAttribute","value":"shaded"},
140-
{"type":"Operator","value":"."},
140+
{"type":"Punctuation","value":"."},
141141
{"type":"NameAttribute","value":"org"},
142-
{"type":"Operator","value":"."},
142+
{"type":"Punctuation","value":"."},
143143
{"type":"NameAttribute","value":"jctools"},
144-
{"type":"Operator","value":"."},
144+
{"type":"Punctuation","value":"."},
145145
{"type":"NameAttribute","value":"util"},
146-
{"type":"Operator","value":"."},
146+
{"type":"Punctuation","value":"."},
147147
{"type":"NameAttribute","value":"UnsafeRefArrayAccess"},
148-
{"type":"Operator","value":"."},
148+
{"type":"Punctuation","value":"."},
149149
{"type":"NameAttribute","value":"class"},
150-
{"type":"Operator","value":")"},
151-
{"type":"Text","value":"\n"},
150+
{"type":"Punctuation","value":")"},
151+
{"type":"TextWhitespace","value":"\n"},
152152
{"type":"KeywordDeclaration","value":"final"},
153-
{"type":"Text","value":" "},
153+
{"type":"TextWhitespace","value":" "},
154154
{"type":"KeywordDeclaration","value":"class"},
155155
{"type":"Text","value":" "},
156156
{"type":"NameClass","value":"TargetUnsafeRefArrayAccess"},
157-
{"type":"Text","value":" "},
158-
{"type":"Operator","value":"{"},
159-
{"type":"Text","value":"\n "},
157+
{"type":"TextWhitespace","value":" "},
158+
{"type":"Punctuation","value":"{"},
159+
{"type":"TextWhitespace","value":"\n "},
160160
{"type":"NameDecorator","value":"@Alias"},
161-
{"type":"Text","value":"\n "},
161+
{"type":"TextWhitespace","value":"\n "},
162162
{"type":"NameDecorator","value":"@RecomputeFieldValue"},
163-
{"type":"Operator","value":"("},
163+
{"type":"Punctuation","value":"("},
164164
{"type":"Name","value":"kind"},
165-
{"type":"Text","value":" "},
165+
{"type":"TextWhitespace","value":" "},
166166
{"type":"Operator","value":"="},
167-
{"type":"Text","value":" "},
167+
{"type":"TextWhitespace","value":" "},
168168
{"type":"Name","value":"RecomputeFieldValue"},
169-
{"type":"Operator","value":"."},
169+
{"type":"Punctuation","value":"."},
170170
{"type":"NameAttribute","value":"Kind"},
171-
{"type":"Operator","value":"."},
171+
{"type":"Punctuation","value":"."},
172172
{"type":"NameAttribute","value":"ArrayIndexShift"},
173-
{"type":"Operator","value":","},
174-
{"type":"Text","value":" "},
173+
{"type":"Punctuation","value":","},
174+
{"type":"TextWhitespace","value":" "},
175175
{"type":"Name","value":"declClass"},
176-
{"type":"Text","value":" "},
176+
{"type":"TextWhitespace","value":" "},
177177
{"type":"Operator","value":"="},
178-
{"type":"Text","value":" "},
178+
{"type":"TextWhitespace","value":" "},
179179
{"type":"Name","value":"Object"},
180-
{"type":"Operator","value":"[]."},
180+
{"type":"Operator","value":"[]"},
181+
{"type":"Punctuation","value":"."},
181182
{"type":"NameAttribute","value":"class"},
182-
{"type":"Operator","value":")"},
183-
{"type":"Text","value":"\n "},
183+
{"type":"Punctuation","value":")"},
184+
{"type":"TextWhitespace","value":"\n "},
184185
{"type":"KeywordDeclaration","value":"public"},
185-
{"type":"Text","value":" "},
186+
{"type":"TextWhitespace","value":" "},
186187
{"type":"KeywordDeclaration","value":"static"},
187-
{"type":"Text","value":" "},
188+
{"type":"TextWhitespace","value":" "},
188189
{"type":"KeywordType","value":"int"},
189-
{"type":"Text","value":" "},
190+
{"type":"TextWhitespace","value":" "},
190191
{"type":"Name","value":"REF_ELEMENT_SHIFT"},
191-
{"type":"Operator","value":";"},
192-
{"type":"Text","value":"\n"},
193-
{"type":"Operator","value":"}"},
194-
{"type":"Text","value":"\n"}
192+
{"type":"Punctuation","value":";"},
193+
{"type":"TextWhitespace","value":"\n\n "},
194+
{"type":"KeywordDeclaration","value":"public"},
195+
{"type":"TextWhitespace","value":" "},
196+
{"type":"KeywordDeclaration","value":"static"},
197+
{"type":"TextWhitespace","value":" "},
198+
{"type":"KeywordType","value":"void"},
199+
{"type":"TextWhitespace","value":" "},
200+
{"type":"NameFunction","value":"test"},
201+
{"type":"Punctuation","value":"()"},
202+
{"type":"TextWhitespace","value":" "},
203+
{"type":"Punctuation","value":"{"},
204+
{"type":"TextWhitespace","value":"\n "},
205+
{"type":"Name","value":"System"},
206+
{"type":"Punctuation","value":"."},
207+
{"type":"NameAttribute","value":"out"},
208+
{"type":"Punctuation","value":"."},
209+
{"type":"NameAttribute","value":"println"},
210+
{"type":"Punctuation","value":"("},
211+
{"type":"LiteralString","value":"\"\"\"\n Hello, world!\n This is a multi-line string!\n It can also contain \"quotes\" and 'apostrophes' without breaking.\n We only need to escape \\\"\"\" inside it.\n \"\"\""},
212+
{"type":"Punctuation","value":");"},
213+
{"type":"TextWhitespace","value":"\n "},
214+
{"type":"Punctuation","value":"}"},
215+
{"type":"TextWhitespace","value":"\n"},
216+
{"type":"Punctuation","value":"}"},
217+
{"type":"TextWhitespace","value":"\n"}
195218
]

‎lexers/testdata/markdown.expected

+7-7
Original file line numberDiff line numberDiff line change
@@ -133,19 +133,19 @@
133133
{"type":"TextWhitespace","value":"\n"},
134134
{"type":"LiteralStringHeredoc","value":"'"},
135135
{"type":"Keyword","value":"return"},
136-
{"type":"Text","value":" "},
136+
{"type":"TextWhitespace","value":" "},
137137
{"type":"Name","value":"Double"},
138-
{"type":"Operator","value":"."},
138+
{"type":"Punctuation","value":"."},
139139
{"type":"NameAttribute","value":"valueOf"},
140-
{"type":"Operator","value":"("},
140+
{"type":"Punctuation","value":"("},
141141
{"type":"Name","value":"Math"},
142-
{"type":"Operator","value":"."},
142+
{"type":"Punctuation","value":"."},
143143
{"type":"NameAttribute","value":"log"},
144-
{"type":"Operator","value":"("},
144+
{"type":"Punctuation","value":"("},
145145
{"type":"Name","value":"input"},
146-
{"type":"Operator","value":"."},
146+
{"type":"Punctuation","value":"."},
147147
{"type":"NameAttribute","value":"doubleValue"},
148-
{"type":"Operator","value":"()));"},
148+
{"type":"Punctuation","value":"()));"},
149149
{"type":"LiteralStringHeredoc","value":"'"},
150150
{"type":"Punctuation","value":";"},
151151
{"type":"TextWhitespace","value":"\n"},

0 commit comments

Comments
 (0)
Please sign in to comment.