diff --git a/lexers/embedded/java.xml b/lexers/embedded/java.xml index eabddee06..3ce33ff6d 100644 --- a/lexers/embedded/java.xml +++ b/lexers/embedded/java.xml @@ -5,116 +5,188 @@ *.java text/x-java true - true - - - - - - - - - - - - + + + + + + + + - + - - + + + + + - + - - + + - - - - + + + + - + - - + + - + - - + + - + - + + + + + - + - - + + - + - - + + - + - - + + + - - + + + - - + + - - + + + + + + + + + - - + + + + + + - + - - + + - + - + - + - + - - + + + + + - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/lexers/testdata/cql.expected b/lexers/testdata/cql.expected index bc5f84984..149338c64 100644 --- a/lexers/testdata/cql.expected +++ b/lexers/testdata/cql.expected @@ -1066,19 +1066,19 @@ {"type":"TextWhitespace","value":"\n"}, {"type":"LiteralStringHeredoc","value":"'"}, {"type":"Keyword","value":"return"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"Double"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"valueOf"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"Math"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"log"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"input"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"doubleValue"}, - {"type":"Operator","value":"()));"}, + {"type":"Punctuation","value":"()));"}, {"type":"LiteralStringHeredoc","value":"'"}, {"type":"Punctuation","value":";"}, {"type":"TextWhitespace","value":"\n"} diff --git a/lexers/testdata/java.actual b/lexers/testdata/java.actual index aed93a8c4..e08b2cc40 100644 --- a/lexers/testdata/java.actual +++ b/lexers/testdata/java.actual @@ -23,4 +23,13 @@ final class TargetUnsafeRefArrayAccess { @Alias @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayIndexShift, declClass = Object[].class) public static int REF_ELEMENT_SHIFT; + + public static void test() { + System.out.println(""" + Hello, world! + This is a multi-line string! + It can also contain "quotes" and 'apostrophes' without breaking. + We only need to escape \""" inside it. + """); + } } diff --git a/lexers/testdata/java.expected b/lexers/testdata/java.expected index 162983fb0..0f01690af 100644 --- a/lexers/testdata/java.expected +++ b/lexers/testdata/java.expected @@ -1,195 +1,218 @@ [ {"type":"KeywordNamespace","value":"package"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"NameNamespace","value":"example"}, - {"type":"Operator","value":";"}, - {"type":"Text","value":"\n\n"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":"\n\n"}, {"type":"KeywordNamespace","value":"import"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Alias"}, - {"type":"Operator","value":";"}, - {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"KeywordNamespace","value":"import"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.RecomputeFieldValue"}, - {"type":"Operator","value":";"}, - {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"KeywordNamespace","value":"import"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Targetclass"}, - {"type":"Operator","value":";"}, - {"type":"Text","value":"\n\n"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":"\n\n"}, {"type":"NameDecorator","value":"@TargetClass"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"className"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"LiteralString","value":"\"io.netty.util.internal.cleanerJava6\""}, - {"type":"Operator","value":")"}, - {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":")"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"KeywordDeclaration","value":"final"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordDeclaration","value":"class"}, {"type":"Text","value":" "}, {"type":"NameClass","value":"TargetCleanerJava6"}, - {"type":"Text","value":" "}, - {"type":"Operator","value":"{"}, - {"type":"Text","value":"\n "}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameDecorator","value":"@Alias"}, - {"type":"Text","value":"\n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameDecorator","value":"@RecomputeFieldValue"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"kind"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"RecomputeFieldValue"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"Kind"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"Fieldoffset"}, - {"type":"Operator","value":","}, - {"type":"Text","value":" "}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"declClassName"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"LiteralString","value":"\"java.nio.DirectBy\""}, - {"type":"Operator","value":")"}, - {"type":"Text","value":"\n "}, + {"type":"Punctuation","value":")"}, + {"type":"TextWhitespace","value":"\n "}, {"type":"KeywordDeclaration","value":"private"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordDeclaration","value":"static"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordType","value":"long"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"CLEANER_FIELD_OFFSET"}, - {"type":"Operator","value":";"}, - {"type":"Text","value":"\n"}, - {"type":"Operator","value":"}"}, - {"type":"Text","value":"\n\n"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"TextWhitespace","value":"\n\n"}, {"type":"NameDecorator","value":"@TargetClass"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"className"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"LiteralString","value":"\"io.netty.util.internal.PlatformDependent0\""}, - {"type":"Operator","value":")"}, - {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":")"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"KeywordDeclaration","value":"final"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordDeclaration","value":"class"}, {"type":"Text","value":" "}, {"type":"NameClass","value":"TargetPlatformDependent0"}, - {"type":"Text","value":" "}, - {"type":"Operator","value":"{"}, - {"type":"Text","value":"\n "}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameDecorator","value":"@Alias"}, - {"type":"Text","value":"\n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameDecorator","value":"@RecomputeFieldValue"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"kind"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"RecomputeFieldValue"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"Kind"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"Fieldoffset"}, - {"type":"Operator","value":","}, - {"type":"Text","value":" "}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"declClassName"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"LiteralString","value":"\"java.nio.Buffer\""}, - {"type":"Operator","value":","}, - {"type":"Text","value":" "}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"rest"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"LiteralString","value":"\"IDK\""}, - {"type":"Operator","value":")"}, - {"type":"Text","value":"\n "}, + {"type":"Punctuation","value":")"}, + {"type":"TextWhitespace","value":"\n "}, {"type":"KeywordDeclaration","value":"private"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordDeclaration","value":"static"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordType","value":"long"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"ADDRESS_FIELD_OFFSET"}, - {"type":"Operator","value":";"}, - {"type":"Text","value":"\n"}, - {"type":"Operator","value":"}"}, - {"type":"Text","value":"\n\n"}, + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"TextWhitespace","value":"\n\n"}, {"type":"NameDecorator","value":"@TargetClass"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"io"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"netty"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"util"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"internal"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"shaded"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"org"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"jctools"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"util"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"UnsafeRefArrayAccess"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"class"}, - {"type":"Operator","value":")"}, - {"type":"Text","value":"\n"}, + {"type":"Punctuation","value":")"}, + {"type":"TextWhitespace","value":"\n"}, {"type":"KeywordDeclaration","value":"final"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordDeclaration","value":"class"}, {"type":"Text","value":" "}, {"type":"NameClass","value":"TargetUnsafeRefArrayAccess"}, - {"type":"Text","value":" "}, - {"type":"Operator","value":"{"}, - {"type":"Text","value":"\n "}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameDecorator","value":"@Alias"}, - {"type":"Text","value":"\n "}, + {"type":"TextWhitespace","value":"\n "}, {"type":"NameDecorator","value":"@RecomputeFieldValue"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"kind"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"RecomputeFieldValue"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"Kind"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"ArrayIndexShift"}, - {"type":"Operator","value":","}, - {"type":"Text","value":" "}, + {"type":"Punctuation","value":","}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"declClass"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Operator","value":"="}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"Object"}, - {"type":"Operator","value":"[]."}, + {"type":"Operator","value":"[]"}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"class"}, - {"type":"Operator","value":")"}, - {"type":"Text","value":"\n "}, + {"type":"Punctuation","value":")"}, + {"type":"TextWhitespace","value":"\n "}, {"type":"KeywordDeclaration","value":"public"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordDeclaration","value":"static"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"KeywordType","value":"int"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"REF_ELEMENT_SHIFT"}, - {"type":"Operator","value":";"}, - {"type":"Text","value":"\n"}, - {"type":"Operator","value":"}"}, - {"type":"Text","value":"\n"} + {"type":"Punctuation","value":";"}, + {"type":"TextWhitespace","value":"\n\n "}, + {"type":"KeywordDeclaration","value":"public"}, + {"type":"TextWhitespace","value":" "}, + {"type":"KeywordDeclaration","value":"static"}, + {"type":"TextWhitespace","value":" "}, + {"type":"KeywordType","value":"void"}, + {"type":"TextWhitespace","value":" "}, + {"type":"NameFunction","value":"test"}, + {"type":"Punctuation","value":"()"}, + {"type":"TextWhitespace","value":" "}, + {"type":"Punctuation","value":"{"}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"Name","value":"System"}, + {"type":"Punctuation","value":"."}, + {"type":"NameAttribute","value":"out"}, + {"type":"Punctuation","value":"."}, + {"type":"NameAttribute","value":"println"}, + {"type":"Punctuation","value":"("}, + {"type":"LiteralString","value":"\"\"\"\n Hello, world!\n This is a multi-line string!\n It can also contain \"quotes\" and 'apostrophes' without breaking.\n We only need to escape \\\"\"\" inside it.\n \"\"\""}, + {"type":"Punctuation","value":");"}, + {"type":"TextWhitespace","value":"\n "}, + {"type":"Punctuation","value":"}"}, + {"type":"TextWhitespace","value":"\n"}, + {"type":"Punctuation","value":"}"}, + {"type":"TextWhitespace","value":"\n"} ] diff --git a/lexers/testdata/markdown.expected b/lexers/testdata/markdown.expected index d4b09cc05..6f5ca68c1 100644 --- a/lexers/testdata/markdown.expected +++ b/lexers/testdata/markdown.expected @@ -133,19 +133,19 @@ {"type":"TextWhitespace","value":"\n"}, {"type":"LiteralStringHeredoc","value":"'"}, {"type":"Keyword","value":"return"}, - {"type":"Text","value":" "}, + {"type":"TextWhitespace","value":" "}, {"type":"Name","value":"Double"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"valueOf"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"Math"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"log"}, - {"type":"Operator","value":"("}, + {"type":"Punctuation","value":"("}, {"type":"Name","value":"input"}, - {"type":"Operator","value":"."}, + {"type":"Punctuation","value":"."}, {"type":"NameAttribute","value":"doubleValue"}, - {"type":"Operator","value":"()));"}, + {"type":"Punctuation","value":"()));"}, {"type":"LiteralStringHeredoc","value":"'"}, {"type":"Punctuation","value":";"}, {"type":"TextWhitespace","value":"\n"},