diff --git a/lexers/embedded/java.xml b/lexers/embedded/java.xml
index eabddee06..3ce33ff6d 100644
--- a/lexers/embedded/java.xml
+++ b/lexers/embedded/java.xml
@@ -5,116 +5,188 @@
*.java
text/x-java
true
- true
-
-
-
-
-
-
-
-
-
-
-
-
+
+
+
+
+
+
+
+
-
+
-
-
+
+
+
+
+
-
+
-
-
+
+
-
-
-
-
+
+
+
+
-
+
-
-
+
+
-
+
-
-
+
+
-
+
-
+
+
+
+
+
-
+
-
-
+
+
-
+
-
-
+
+
-
+
-
-
+
+
+
-
-
+
+
+
-
-
+
+
-
-
+
+
+
+
+
+
+
+
+
-
-
+
+
+
+
+
+
-
+
-
-
+
+
-
+
-
+
-
+
-
+
-
-
+
+
+
+
+
-
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/lexers/testdata/cql.expected b/lexers/testdata/cql.expected
index bc5f84984..149338c64 100644
--- a/lexers/testdata/cql.expected
+++ b/lexers/testdata/cql.expected
@@ -1066,19 +1066,19 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"LiteralStringHeredoc","value":"'"},
{"type":"Keyword","value":"return"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"Double"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"valueOf"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"Math"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"log"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"input"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"doubleValue"},
- {"type":"Operator","value":"()));"},
+ {"type":"Punctuation","value":"()));"},
{"type":"LiteralStringHeredoc","value":"'"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n"}
diff --git a/lexers/testdata/java.actual b/lexers/testdata/java.actual
index aed93a8c4..e08b2cc40 100644
--- a/lexers/testdata/java.actual
+++ b/lexers/testdata/java.actual
@@ -23,4 +23,13 @@ final class TargetUnsafeRefArrayAccess {
@Alias
@RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayIndexShift, declClass = Object[].class)
public static int REF_ELEMENT_SHIFT;
+
+ public static void test() {
+ System.out.println("""
+ Hello, world!
+ This is a multi-line string!
+ It can also contain "quotes" and 'apostrophes' without breaking.
+ We only need to escape \""" inside it.
+ """);
+ }
}
diff --git a/lexers/testdata/java.expected b/lexers/testdata/java.expected
index 162983fb0..0f01690af 100644
--- a/lexers/testdata/java.expected
+++ b/lexers/testdata/java.expected
@@ -1,195 +1,218 @@
[
{"type":"KeywordNamespace","value":"package"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"NameNamespace","value":"example"},
- {"type":"Operator","value":";"},
- {"type":"Text","value":"\n\n"},
+ {"type":"Punctuation","value":";"},
+ {"type":"TextWhitespace","value":"\n\n"},
{"type":"KeywordNamespace","value":"import"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Alias"},
- {"type":"Operator","value":";"},
- {"type":"Text","value":"\n"},
+ {"type":"Punctuation","value":";"},
+ {"type":"TextWhitespace","value":"\n"},
{"type":"KeywordNamespace","value":"import"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"NameNamespace","value":"com.oracle.svm.core.annotate.RecomputeFieldValue"},
- {"type":"Operator","value":";"},
- {"type":"Text","value":"\n"},
+ {"type":"Punctuation","value":";"},
+ {"type":"TextWhitespace","value":"\n"},
{"type":"KeywordNamespace","value":"import"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Targetclass"},
- {"type":"Operator","value":";"},
- {"type":"Text","value":"\n\n"},
+ {"type":"Punctuation","value":";"},
+ {"type":"TextWhitespace","value":"\n\n"},
{"type":"NameDecorator","value":"@TargetClass"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"className"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"\"io.netty.util.internal.cleanerJava6\""},
- {"type":"Operator","value":")"},
- {"type":"Text","value":"\n"},
+ {"type":"Punctuation","value":")"},
+ {"type":"TextWhitespace","value":"\n"},
{"type":"KeywordDeclaration","value":"final"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordDeclaration","value":"class"},
{"type":"Text","value":" "},
{"type":"NameClass","value":"TargetCleanerJava6"},
- {"type":"Text","value":" "},
- {"type":"Operator","value":"{"},
- {"type":"Text","value":"\n "},
+ {"type":"TextWhitespace","value":" "},
+ {"type":"Punctuation","value":"{"},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"NameDecorator","value":"@Alias"},
- {"type":"Text","value":"\n "},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"NameDecorator","value":"@RecomputeFieldValue"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"kind"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"RecomputeFieldValue"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"Kind"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"Fieldoffset"},
- {"type":"Operator","value":","},
- {"type":"Text","value":" "},
+ {"type":"Punctuation","value":","},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"declClassName"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"\"java.nio.DirectBy\""},
- {"type":"Operator","value":")"},
- {"type":"Text","value":"\n "},
+ {"type":"Punctuation","value":")"},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"KeywordDeclaration","value":"private"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordDeclaration","value":"static"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"long"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"CLEANER_FIELD_OFFSET"},
- {"type":"Operator","value":";"},
- {"type":"Text","value":"\n"},
- {"type":"Operator","value":"}"},
- {"type":"Text","value":"\n\n"},
+ {"type":"Punctuation","value":";"},
+ {"type":"TextWhitespace","value":"\n"},
+ {"type":"Punctuation","value":"}"},
+ {"type":"TextWhitespace","value":"\n\n"},
{"type":"NameDecorator","value":"@TargetClass"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"className"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"\"io.netty.util.internal.PlatformDependent0\""},
- {"type":"Operator","value":")"},
- {"type":"Text","value":"\n"},
+ {"type":"Punctuation","value":")"},
+ {"type":"TextWhitespace","value":"\n"},
{"type":"KeywordDeclaration","value":"final"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordDeclaration","value":"class"},
{"type":"Text","value":" "},
{"type":"NameClass","value":"TargetPlatformDependent0"},
- {"type":"Text","value":" "},
- {"type":"Operator","value":"{"},
- {"type":"Text","value":"\n "},
+ {"type":"TextWhitespace","value":" "},
+ {"type":"Punctuation","value":"{"},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"NameDecorator","value":"@Alias"},
- {"type":"Text","value":"\n "},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"NameDecorator","value":"@RecomputeFieldValue"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"kind"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"RecomputeFieldValue"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"Kind"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"Fieldoffset"},
- {"type":"Operator","value":","},
- {"type":"Text","value":" "},
+ {"type":"Punctuation","value":","},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"declClassName"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"\"java.nio.Buffer\""},
- {"type":"Operator","value":","},
- {"type":"Text","value":" "},
+ {"type":"Punctuation","value":","},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"rest"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"LiteralString","value":"\"IDK\""},
- {"type":"Operator","value":")"},
- {"type":"Text","value":"\n "},
+ {"type":"Punctuation","value":")"},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"KeywordDeclaration","value":"private"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordDeclaration","value":"static"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"long"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"ADDRESS_FIELD_OFFSET"},
- {"type":"Operator","value":";"},
- {"type":"Text","value":"\n"},
- {"type":"Operator","value":"}"},
- {"type":"Text","value":"\n\n"},
+ {"type":"Punctuation","value":";"},
+ {"type":"TextWhitespace","value":"\n"},
+ {"type":"Punctuation","value":"}"},
+ {"type":"TextWhitespace","value":"\n\n"},
{"type":"NameDecorator","value":"@TargetClass"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"io"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"netty"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"util"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"internal"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"shaded"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"org"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"jctools"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"util"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"UnsafeRefArrayAccess"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"class"},
- {"type":"Operator","value":")"},
- {"type":"Text","value":"\n"},
+ {"type":"Punctuation","value":")"},
+ {"type":"TextWhitespace","value":"\n"},
{"type":"KeywordDeclaration","value":"final"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordDeclaration","value":"class"},
{"type":"Text","value":" "},
{"type":"NameClass","value":"TargetUnsafeRefArrayAccess"},
- {"type":"Text","value":" "},
- {"type":"Operator","value":"{"},
- {"type":"Text","value":"\n "},
+ {"type":"TextWhitespace","value":" "},
+ {"type":"Punctuation","value":"{"},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"NameDecorator","value":"@Alias"},
- {"type":"Text","value":"\n "},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"NameDecorator","value":"@RecomputeFieldValue"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"kind"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"RecomputeFieldValue"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"Kind"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"ArrayIndexShift"},
- {"type":"Operator","value":","},
- {"type":"Text","value":" "},
+ {"type":"Punctuation","value":","},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"declClass"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Operator","value":"="},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"Object"},
- {"type":"Operator","value":"[]."},
+ {"type":"Operator","value":"[]"},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"class"},
- {"type":"Operator","value":")"},
- {"type":"Text","value":"\n "},
+ {"type":"Punctuation","value":")"},
+ {"type":"TextWhitespace","value":"\n "},
{"type":"KeywordDeclaration","value":"public"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordDeclaration","value":"static"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"KeywordType","value":"int"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"REF_ELEMENT_SHIFT"},
- {"type":"Operator","value":";"},
- {"type":"Text","value":"\n"},
- {"type":"Operator","value":"}"},
- {"type":"Text","value":"\n"}
+ {"type":"Punctuation","value":";"},
+ {"type":"TextWhitespace","value":"\n\n "},
+ {"type":"KeywordDeclaration","value":"public"},
+ {"type":"TextWhitespace","value":" "},
+ {"type":"KeywordDeclaration","value":"static"},
+ {"type":"TextWhitespace","value":" "},
+ {"type":"KeywordType","value":"void"},
+ {"type":"TextWhitespace","value":" "},
+ {"type":"NameFunction","value":"test"},
+ {"type":"Punctuation","value":"()"},
+ {"type":"TextWhitespace","value":" "},
+ {"type":"Punctuation","value":"{"},
+ {"type":"TextWhitespace","value":"\n "},
+ {"type":"Name","value":"System"},
+ {"type":"Punctuation","value":"."},
+ {"type":"NameAttribute","value":"out"},
+ {"type":"Punctuation","value":"."},
+ {"type":"NameAttribute","value":"println"},
+ {"type":"Punctuation","value":"("},
+ {"type":"LiteralString","value":"\"\"\"\n Hello, world!\n This is a multi-line string!\n It can also contain \"quotes\" and 'apostrophes' without breaking.\n We only need to escape \\\"\"\" inside it.\n \"\"\""},
+ {"type":"Punctuation","value":");"},
+ {"type":"TextWhitespace","value":"\n "},
+ {"type":"Punctuation","value":"}"},
+ {"type":"TextWhitespace","value":"\n"},
+ {"type":"Punctuation","value":"}"},
+ {"type":"TextWhitespace","value":"\n"}
]
diff --git a/lexers/testdata/markdown.expected b/lexers/testdata/markdown.expected
index d4b09cc05..6f5ca68c1 100644
--- a/lexers/testdata/markdown.expected
+++ b/lexers/testdata/markdown.expected
@@ -133,19 +133,19 @@
{"type":"TextWhitespace","value":"\n"},
{"type":"LiteralStringHeredoc","value":"'"},
{"type":"Keyword","value":"return"},
- {"type":"Text","value":" "},
+ {"type":"TextWhitespace","value":" "},
{"type":"Name","value":"Double"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"valueOf"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"Math"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"log"},
- {"type":"Operator","value":"("},
+ {"type":"Punctuation","value":"("},
{"type":"Name","value":"input"},
- {"type":"Operator","value":"."},
+ {"type":"Punctuation","value":"."},
{"type":"NameAttribute","value":"doubleValue"},
- {"type":"Operator","value":"()));"},
+ {"type":"Punctuation","value":"()));"},
{"type":"LiteralStringHeredoc","value":"'"},
{"type":"Punctuation","value":";"},
{"type":"TextWhitespace","value":"\n"},