Fix Java lexer (synced with latest Pygments).

Fixes #297.
alecthomas · Nov 5, 2019 · 3aaf3e5 · 3aaf3e5
1 parent bbc59ac
commit 3aaf3e5
Show file tree

Hide file tree

Showing 8 changed files with 240 additions and 22 deletions.
diff --git a/go.mod b/go.mod
@@ -14,9 +14,8 @@ require (
 	github.com/gorilla/mux v1.7.3
 	github.com/mattn/go-colorable v0.0.9
 	github.com/mattn/go-isatty v0.0.4
-	github.com/pkg/errors v0.8.1
 	github.com/sergi/go-diff v1.0.0 // indirect
-	github.com/stretchr/testify v1.3.0
+	github.com/stretchr/testify v1.3.0 // indirect
 	golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect
 )
 

diff --git a/lexers/c/cpp_test.go b/lexers/c/cpp_test.go
@@ -3,7 +3,7 @@ package c_test
 import (
 	"testing"
 
-	"github.com/stretchr/testify/require"
+	"github.com/alecthomas/assert"
 
 	"github.com/alecthomas/chroma"
 	"github.com/alecthomas/chroma/lexers/c"
@@ -17,7 +17,7 @@ double c =     0011111111010011001100110011001100110011001100110011001100110011;
 double a + b = 0011111111010011001100110011001100110011001100110011001100110100; // Note that this is not quite equal to the "canonical" 0.3!a
 `
 	it, err := c.CPP.Tokenise(nil, input)
-	require.NoError(t, err)
+	assert.NoError(t, err)
 	for {
 		token := it()
 		if token == chroma.EOF {

diff --git a/lexers/j/java.go b/lexers/j/java.go
@@ -20,7 +20,7 @@ var Java = internal.Register(MustNewLexer(
 			{`//.*?\n`, CommentSingle, nil},
 			{`/\*.*?\*/`, CommentMultiline, nil},
 			{`(assert|break|case|catch|continue|default|do|else|finally|for|if|goto|instanceof|new|return|switch|this|throw|try|while)\b`, Keyword, nil},
-			{`((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)((?:[^\W\d]|\$)[\w$]*)(\s*)`, ByGroups(UsingSelf("root"), NameFunction, Text, Operator), nil},
+			{`((?:(?:[^\W\d]|\$)[\w.\[\]$<>]*\s+)+?)((?:[^\W\d]|\$)[\w$]*)(\s*)(\()`, ByGroups(UsingSelf("root"), NameFunction, Text, Operator), nil},
 			{`@[^\W\d][\w.]*`, NameDecorator, nil},
 			{`(abstract|const|enum|extends|final|implements|native|private|protected|public|static|strictfp|super|synchronized|throws|transient|volatile)\b`, KeywordDeclaration, nil},
 			{`(boolean|byte|char|double|float|int|long|short|void)\b`, KeywordType, nil},
@@ -30,16 +30,15 @@ var Java = internal.Register(MustNewLexer(
 			{`(import(?:\s+static)?)(\s+)`, ByGroups(KeywordNamespace, Text), Push("import")},
 			{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
 			{`'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'`, LiteralStringChar, nil},
-			{`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Punctuation, NameAttribute), nil},
+			{`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Operator, NameAttribute), nil},
 			{`^\s*([^\W\d]|\$)[\w$]*:`, NameLabel, nil},
 			{`([^\W\d]|\$)[\w$]*`, Name, nil},
 			{`([0-9][0-9_]*\.([0-9][0-9_]*)?|\.[0-9][0-9_]*)([eE][+\-]?[0-9][0-9_]*)?[fFdD]?|[0-9][eE][+\-]?[0-9][0-9_]*[fFdD]?|[0-9]([eE][+\-]?[0-9][0-9_]*)?[fFdD]|0[xX]([0-9a-fA-F][0-9a-fA-F_]*\.?|([0-9a-fA-F][0-9a-fA-F_]*)?\.[0-9a-fA-F][0-9a-fA-F_]*)[pP][+\-]?[0-9][0-9_]*[fFdD]?`, LiteralNumberFloat, nil},
 			{`0[xX][0-9a-fA-F][0-9a-fA-F_]*[lL]?`, LiteralNumberHex, nil},
 			{`0[bB][01][01_]*[lL]?`, LiteralNumberBin, nil},
 			{`0[0-7_]+[lL]?`, LiteralNumberOct, nil},
 			{`0|[1-9][0-9_]*[lL]?`, LiteralNumberInteger, nil},
-			{`[~^*!%&<>|+=:/?-]`, Operator, nil},
-			{`[\[\](){};,.]`, Punctuation, nil},
+			{`[~^*!%&\[\](){}<>|+=:;,./?-]`, Operator, nil},
 			{`\n`, Text, nil},
 		},
 		"class": {

diff --git a/lexers/testdata/cql.expected b/lexers/testdata/cql.expected
@@ -1068,17 +1068,17 @@
   {"type":"Keyword","value":"return"},
   {"type":"Text","value":" "},
   {"type":"Name","value":"Double"},
-  {"type":"Punctuation","value":"."},
+  {"type":"Operator","value":"."},
   {"type":"NameAttribute","value":"valueOf"},
-  {"type":"Punctuation","value":"("},
+  {"type":"Operator","value":"("},
   {"type":"Name","value":"Math"},
-  {"type":"Punctuation","value":"."},
+  {"type":"Operator","value":"."},
   {"type":"NameAttribute","value":"log"},
-  {"type":"Punctuation","value":"("},
+  {"type":"Operator","value":"("},
   {"type":"Name","value":"input"},
-  {"type":"Punctuation","value":"."},
+  {"type":"Operator","value":"."},
   {"type":"NameAttribute","value":"doubleValue"},
-  {"type":"Punctuation","value":"()));"},
+  {"type":"Operator","value":"()));"},
   {"type":"LiteralStringHeredoc","value":"'"},
   {"type":"Punctuation","value":";"},
   {"type":"TextWhitespace","value":"\n"}

diff --git a/lexers/testdata/java.actual b/lexers/testdata/java.actual
@@ -0,0 +1,26 @@
+package example;
+
+import com.oracle.svm.core.annotate.Alias;
+import com.oracle.svm.core.annotate.RecomputeFieldValue;
+import com.oracle.svm.core.annotate.Targetclass;
+
+@TargetClass(className = "io.netty.util.internal.cleanerJava6")
+final class TargetCleanerJava6 {
+    @Alias
+    @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Fieldoffset, declClassName = "java.nio.DirectBy")
+    private static long CLEANER_FIELD_OFFSET;
+}
+
+@TargetClass(className = "io.netty.util.internal.PlatformDependent0")
+final class TargetPlatformDependent0 {
+    @Alias
+    @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.Fieldoffset, declClassName = "java.nio.Buffer", rest = "IDK")
+    private static long ADDRESS_FIELD_OFFSET;
+}
+
+@TargetClass(io.netty.util.internal.shaded.org.jctools.util.UnsafeRefArrayAccess.class)
+final class TargetUnsafeRefArrayAccess {
+    @Alias
+    @RecomputeFieldValue(kind = RecomputeFieldValue.Kind.ArrayIndexShift, declClass = Object[].class)
+    public static int REF_ELEMENT_SHIFT;
+}
diff --git a/lexers/testdata/java.expected b/lexers/testdata/java.expected
@@ -0,0 +1,195 @@
+[
+  {"type":"KeywordNamespace","value":"package"},
+  {"type":"Text","value":" "},
+  {"type":"NameNamespace","value":"example"},
+  {"type":"Operator","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"KeywordNamespace","value":"import"},
+  {"type":"Text","value":" "},
+  {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Alias"},
+  {"type":"Operator","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordNamespace","value":"import"},
+  {"type":"Text","value":" "},
+  {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.RecomputeFieldValue"},
+  {"type":"Operator","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordNamespace","value":"import"},
+  {"type":"Text","value":" "},
+  {"type":"NameNamespace","value":"com.oracle.svm.core.annotate.Targetclass"},
+  {"type":"Operator","value":";"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"NameDecorator","value":"@TargetClass"},
+  {"type":"Operator","value":"("},
+  {"type":"Name","value":"className"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"io.netty.util.internal.cleanerJava6\""},
+  {"type":"Operator","value":")"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordDeclaration","value":"final"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordDeclaration","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TargetCleanerJava6"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameDecorator","value":"@Alias"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameDecorator","value":"@RecomputeFieldValue"},
+  {"type":"Operator","value":"("},
+  {"type":"Name","value":"kind"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"RecomputeFieldValue"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"Kind"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"Fieldoffset"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"declClassName"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"java.nio.DirectBy\""},
+  {"type":"Operator","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordDeclaration","value":"private"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordDeclaration","value":"static"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"long"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"CLEANER_FIELD_OFFSET"},
+  {"type":"Operator","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Operator","value":"}"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"NameDecorator","value":"@TargetClass"},
+  {"type":"Operator","value":"("},
+  {"type":"Name","value":"className"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"io.netty.util.internal.PlatformDependent0\""},
+  {"type":"Operator","value":")"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordDeclaration","value":"final"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordDeclaration","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TargetPlatformDependent0"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameDecorator","value":"@Alias"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameDecorator","value":"@RecomputeFieldValue"},
+  {"type":"Operator","value":"("},
+  {"type":"Name","value":"kind"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"RecomputeFieldValue"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"Kind"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"Fieldoffset"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"declClassName"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"java.nio.Buffer\""},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"rest"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"LiteralString","value":"\"IDK\""},
+  {"type":"Operator","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordDeclaration","value":"private"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordDeclaration","value":"static"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"long"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"ADDRESS_FIELD_OFFSET"},
+  {"type":"Operator","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Operator","value":"}"},
+  {"type":"Text","value":"\n\n"},
+  {"type":"NameDecorator","value":"@TargetClass"},
+  {"type":"Operator","value":"("},
+  {"type":"Name","value":"io"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"netty"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"util"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"internal"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"shaded"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"org"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"jctools"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"util"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"UnsafeRefArrayAccess"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"class"},
+  {"type":"Operator","value":")"},
+  {"type":"Text","value":"\n"},
+  {"type":"KeywordDeclaration","value":"final"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordDeclaration","value":"class"},
+  {"type":"Text","value":" "},
+  {"type":"NameClass","value":"TargetUnsafeRefArrayAccess"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"{"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameDecorator","value":"@Alias"},
+  {"type":"Text","value":"\n    "},
+  {"type":"NameDecorator","value":"@RecomputeFieldValue"},
+  {"type":"Operator","value":"("},
+  {"type":"Name","value":"kind"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"RecomputeFieldValue"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"Kind"},
+  {"type":"Operator","value":"."},
+  {"type":"NameAttribute","value":"ArrayIndexShift"},
+  {"type":"Operator","value":","},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"declClass"},
+  {"type":"Text","value":" "},
+  {"type":"Operator","value":"="},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"Object"},
+  {"type":"Operator","value":"[]."},
+  {"type":"NameAttribute","value":"class"},
+  {"type":"Operator","value":")"},
+  {"type":"Text","value":"\n    "},
+  {"type":"KeywordDeclaration","value":"public"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordDeclaration","value":"static"},
+  {"type":"Text","value":" "},
+  {"type":"KeywordType","value":"int"},
+  {"type":"Text","value":" "},
+  {"type":"Name","value":"REF_ELEMENT_SHIFT"},
+  {"type":"Operator","value":";"},
+  {"type":"Text","value":"\n"},
+  {"type":"Operator","value":"}"},
+  {"type":"Text","value":"\n"}
+]
diff --git a/lexers/testdata/markdown.expected b/lexers/testdata/markdown.expected
@@ -119,17 +119,17 @@
   {"type":"Keyword","value":"return"},
   {"type":"Text","value":" "},
   {"type":"Name","value":"Double"},
-  {"type":"Punctuation","value":"."},
+  {"type":"Operator","value":"."},
   {"type":"NameAttribute","value":"valueOf"},
-  {"type":"Punctuation","value":"("},
+  {"type":"Operator","value":"("},
   {"type":"Name","value":"Math"},
-  {"type":"Punctuation","value":"."},
+  {"type":"Operator","value":"."},
   {"type":"NameAttribute","value":"log"},
-  {"type":"Punctuation","value":"("},
+  {"type":"Operator","value":"("},
   {"type":"Name","value":"input"},
-  {"type":"Punctuation","value":"."},
+  {"type":"Operator","value":"."},
   {"type":"NameAttribute","value":"doubleValue"},
-  {"type":"Punctuation","value":"()));"},
+  {"type":"Operator","value":"()));"},
   {"type":"LiteralStringHeredoc","value":"'"},
   {"type":"Punctuation","value":";"},
   {"type":"TextWhitespace","value":"\n"},

diff --git a/regexp_test.go b/regexp_test.go
@@ -4,7 +4,6 @@ import (
 	"testing"
 
 	"github.com/alecthomas/assert"
-	"github.com/stretchr/testify/require"
 )
 
 func TestNewlineAtEndOfFile(t *testing.T) {
@@ -40,7 +39,7 @@ func TestMatchingAtStart(t *testing.T) {
 	}))
 	it, err := l.Tokenise(nil, `-module ->`)
 	assert.NoError(t, err)
-	require.Equal(t,
+	assert.Equal(t,
 		[]Token{{Punctuation, "-"}, {NameEntity, "module"}, {Whitespace, " "}, {Operator, "->"}},
 		it.Tokens())
 }