Emit error tokens when there's a group mismatch.

Also don't panic/recover, as we no longer use panic to report "real"
errors.

Fixes #295.

formatters/html/html.go

@@ -91,11 +91,6 @@ func (h highlightRanges) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }
 func (h highlightRanges) Less(i, j int) bool { return h[i][0] < h[j][0] }
 
 func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Iterator) (err error) {
-	defer func() {
-		if perr := recover(); perr != nil {
-			err = perr.(error)
-		}
-	}()
 	return f.writeHTML(w, style, iterator.Tokens())
 }
 

formatters/html/html_test.go

@@ -2,7 +2,6 @@ package html
 
 import (
 	"bytes"
-	"errors"
 	"io/ioutil"
 	"strings"
 	"testing"
@@ -50,14 +49,6 @@ func TestSplitTokensIntoLines(t *testing.T) {
 	assert.Equal(t, expected, actual)
 }
 
-func TestIteratorPanicRecovery(t *testing.T) {
-	it := func() chroma.Token {
-		panic(errors.New("bad"))
-	}
-	err := New().Format(ioutil.Discard, styles.Fallback, it)
-	assert.Error(t, err)
-}
-
 func TestFormatterStyleToCSS(t *testing.T) {
 	builder := styles.Get("github").Builder()
 	builder.Add(chroma.LineHighlight, "bg:#ffffcc")

formatters/svg/svg.go

@@ -62,11 +62,6 @@ type Formatter struct {
 }
 
 func (f *Formatter) Format(w io.Writer, style *chroma.Style, iterator chroma.Iterator) (err error) {
-	defer func() {
-		if perr := recover(); perr != nil {
-			err = perr.(error)
-		}
-	}()
 	f.writeSVG(w, style, iterator.Tokens())
 	return err
 }

formatters/tty_indexed.go

@@ -225,11 +225,6 @@ type indexedTTYFormatter struct {
 }
 
 func (c *indexedTTYFormatter) Format(w io.Writer, style *chroma.Style, it chroma.Iterator) (err error) {
-	defer func() {
-		if perr := recover(); perr != nil {
-			err = perr.(error)
-		}
-	}()
 	theme := styleToEscapeSequence(c.table, style)
 	for token := it(); token != chroma.EOF; token = it() {
 		// TODO: Cache token lookups?

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/gorilla/mux v1.7.3
 	github.com/mattn/go-colorable v0.0.9
 	github.com/mattn/go-isatty v0.0.4
+	github.com/pkg/errors v0.8.1
 	github.com/sergi/go-diff v1.0.0 // indirect
 	github.com/stretchr/testify v1.3.0
 	golang.org/x/sys v0.0.0-20181128092732-4ed8d59d0b35 // indirect

lexers/b/ballerina.go

@@ -25,7 +25,7 @@ var Ballerina = internal.Register(MustNewLexer(
 			{`(annotation|bind|but|endpoint|error|function|object|private|public|returns|service|type|var|with|worker)\b`, KeywordDeclaration, nil},
 			{`(boolean|byte|decimal|float|int|json|map|nil|record|string|table|xml)\b`, KeywordType, nil},
 			{`(true|false|null)\b`, KeywordConstant, nil},
-			{`import(\s+)`, ByGroups(KeywordNamespace, Text), Push("import")},
+			{`(import)(\s+)`, ByGroups(KeywordNamespace, Text), Push("import")},
 			{`"(\\\\|\\"|[^"])*"`, LiteralString, nil},
 			{`'\\.'|'[^\\]'|'\\u[0-9a-fA-F]{4}'`, LiteralStringChar, nil},
 			{`(\.)((?:[^\W\d]|\$)[\w$]*)`, ByGroups(Operator, NameAttribute), nil},

lexers/d/docker.go

@@ -22,7 +22,7 @@ var Docker = internal.Register(MustNewLexer(
 			{`(ONBUILD)((?:\s*\\?\s*))`, ByGroups(Keyword, Using(b.Bash)), nil},
 			{`(HEALTHCHECK)(((?:\s*\\?\s*)--\w+=\w+(?:\s*\\?\s*))*)`, ByGroups(Keyword, Using(b.Bash)), nil},
 			{`(VOLUME|ENTRYPOINT|CMD|SHELL)((?:\s*\\?\s*))(\[.*?\])`, ByGroups(Keyword, Using(b.Bash), Using(j.JSON)), nil},
-			{`(LABEL|ENV|ARG)(((?:\s*\\?\s*)\w+=\w+(?:\s*\\?\s*))*)`, ByGroups(Keyword, Using(b.Bash)), nil},
+			{`(LABEL|ENV|ARG)((?:(?:\s*\\?\s*)\w+=\w+(?:\s*\\?\s*))*)`, ByGroups(Keyword, Using(b.Bash)), nil},
 			{`((?:FROM|MAINTAINER|EXPOSE|WORKDIR|USER|STOPSIGNAL)|VOLUME)\b(.*)`, ByGroups(Keyword, LiteralString), nil},
 			{`((?:RUN|CMD|ENTRYPOINT|ENV|ARG|LABEL|ADD|COPY))`, Keyword, nil},
 			{`(.*\\\n)*.+`, Using(b.Bash), nil},

lexers/testdata/ballerina.expected

@@ -1,5 +1,6 @@
 [
-  {"type":"KeywordNamespace","value":" "},
+  {"type":"KeywordNamespace","value":"import"},
+  {"type":"Text","value":" "},
   {"type":"NameNamespace","value":"ballerina"},
   {"type":"Operator","value":"/"},
   {"type":"Name","value":"http"},

regexp.go

@@ -35,8 +35,13 @@ func ByGroups(emitters ...Emitter) Emitter {
 	return EmitterFunc(func(groups []string, lexer Lexer) Iterator {
 		iterators := make([]Iterator, 0, len(groups)-1)
 		// NOTE: If this panics, there is a mismatch with groups
-		for i, group := range groups[1:] {
-			iterators = append(iterators, emitters[i].Emit([]string{group}, lexer))
+		if len(emitters) != len(groups)-1 {
+			iterators = append(iterators, Error.Emit(groups, lexer))
+			// panic(errors.Errorf("number of groups %q does not match number of emitters %v", groups, emitters))
+		} else {
+			for i, group := range groups[1:] {
+				iterators = append(iterators, emitters[i].Emit([]string{group}, lexer))
+			}
 		}
 		return Concaterator(iterators...)
 	})