Skip to content

Commit

Permalink
Fix bufio.Scanner: token too long
Browse files Browse the repository at this point in the history
  • Loading branch information
bd82 committed Oct 5, 2021
1 parent c40e9c6 commit 25becca
Show file tree
Hide file tree
Showing 3 changed files with 56 additions and 10 deletions.
2 changes: 2 additions & 0 deletions fixtures/long.env

Large diffs are not rendered by default.

40 changes: 30 additions & 10 deletions godotenv.go
Original file line number Diff line number Diff line change
Expand Up @@ -99,16 +99,7 @@ func Read(filenames ...string) (envMap map[string]string, err error) {
// Parse reads an env file from io.Reader, returning a map of keys and values.
func Parse(r io.Reader) (envMap map[string]string, err error) {
envMap = make(map[string]string)

var lines []string
scanner := bufio.NewScanner(r)
for scanner.Scan() {
lines = append(lines, scanner.Text())
}

if err = scanner.Err(); err != nil {
return
}
lines, err := ReadAllLines(r)

for _, fullLine := range lines {
if !isIgnoredLine(fullLine) {
Expand All @@ -124,6 +115,35 @@ func Parse(r io.Reader) (envMap map[string]string, err error) {
return
}

// ReadAllLines will convert the Reader into an array of strings
// one for each line in the original text.
func ReadAllLines(r io.Reader) (lines []string, err error) {
bf := bufio.NewReader(r)
linePart, isPrefix, err := bf.ReadLine()

for err == nil {
lineStr := string(linePart)

// handling of lines larger than the default buffer size
for err == nil && isPrefix == true {
linePart, isPrefix, err = bf.ReadLine()
if err != nil {
break
}
lineStr += string(linePart)
}

// `err == nil` is used to skip lines which caused an error
if isPrefix == false && err == nil {
lines = append(lines, lineStr)
}

linePart, isPrefix, err = bf.ReadLine()
}

return
}

//Unmarshal reads an env file from a string, returning a map of keys and values.
func Unmarshal(str string) (envMap map[string]string, err error) {
return Parse(strings.NewReader(str))
Expand Down
24 changes: 24 additions & 0 deletions godotenv_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,30 @@ func TestReadPlainEnv(t *testing.T) {
}
}

func TestReadPlainEnvWithLongLine(t *testing.T) {
envFileName := "fixtures/long.env"
expectedBigValue := strings.Repeat("abcd", 20000)
expectedValues := map[string]string{
"BIG_VALUE": expectedBigValue,
"SMALL_VALUE": "abcd",
}

envMap, err := Read(envFileName)
if err != nil {
t.Error("Error reading file -> " + err.Error() )
}

if len(envMap) != len(expectedValues) {
t.Error("Didn't get the right size map back -->")
}

for key, value := range expectedValues {
if envMap[key] != value {
t.Error("Read got one of the keys wrong")
}
}
}

func TestParse(t *testing.T) {
envMap, err := Parse(bytes.NewReader([]byte("ONE=1\nTWO='2'\nTHREE = \"3\"")))
expectedValues := map[string]string{
Expand Down

0 comments on commit 25becca

Please sign in to comment.