diff --git a/cmd/sops/subcommand/exec/exec.go b/cmd/sops/subcommand/exec/exec.go index cd8d33be5..669335dbd 100644 --- a/cmd/sops/subcommand/exec/exec.go +++ b/cmd/sops/subcommand/exec/exec.go @@ -1,13 +1,14 @@ package exec import ( - "bytes" + "fmt" "io/ioutil" "os" "runtime" "strings" "go.mozilla.org/sops/v3/logging" + "go.mozilla.org/sops/v3/stores/dotenv" "github.com/sirupsen/logrus" ) @@ -85,15 +86,18 @@ func ExecWithEnv(opts ExecOpts) error { } env := os.Environ() - lines := bytes.Split(opts.Plaintext, []byte("\n")) - for _, line := range lines { - if len(line) == 0 { - continue - } - if line[0] == '#' { + store := dotenv.Store{} + + branches, err := store.LoadPlainFile(opts.Plaintext) + if err != nil { + log.Fatal(err) + } + + for _, item := range branches[0] { + if item.Value == nil { continue } - env = append(env, string(line)) + env = append(env, fmt.Sprintf("%s=%s", item.Key, item.Value)) } cmd := BuildCommand(opts.Command) diff --git a/stores/dotenv/parser.go b/stores/dotenv/parser.go new file mode 100644 index 000000000..ff9b7b52a --- /dev/null +++ b/stores/dotenv/parser.go @@ -0,0 +1,311 @@ +package dotenv + +// The dotenv parser is designed around the following rules: +// +// Comments: +// +// * Comments may be written by starting a line with the `#` character. +// End-of-line comments are not currently supported, as there is no way to +// encode a comment's position in a `sops.TreeItem`. +// +// Newline handling: +// +// * If a value is unquoted or single-quoted and contains the character +// sequence `\n` (`0x5c6e`), it IS NOT decoded to a line feed (`0x0a`). +// +// * If a value is double-quoted and contains the character sequence `\n` +// (`0x5c6e`), it IS decoded to a line feed (`0x0a`). +// +// Whitespace trimming: +// +// * For comments, the whitespace immediately after the `#` character and any +// trailing whitespace is trimmed. +// +// * If a value is unquoted and contains any leading or trailing whitespace, it +// is trimmed. +// +// * If a value is either single- or double-quoted and contains any leading or +// trailing whitespace, it is left untrimmed. +// +// Quotation handling: +// +// * If a value is surrounded by single- or double-quotes, the quotation marks +// are interpreted and not included in the value. +// +// * Any number of single-quote characters may appear in a double-quoted +// value, or within a single-quoted value if they are escaped (i.e., +// `'foo\'bar'`). +// +// * Any number of double-quote characters may appear in a single-quoted +// value, or within a double-quoted value if they are escaped (i.e., +// `"foo\"bar"`). + +import ( + "bytes" + "fmt" + "io" + "regexp" + "strings" + + "go.mozilla.org/sops/v3" +) + +var KeyRegexp = regexp.MustCompile(`^[A-Za-z_]+[A-Za-z0-9_]*$`) + +func parse(data []byte) (items []sops.TreeItem, err error) { + reader := bytes.NewReader(data) + + for { + var b byte + var item *sops.TreeItem + + b, err = reader.ReadByte() + + if err != nil { + break + } + + if isWhitespace(b) { + continue + } + + if b == '#' { + item, err = parseComment(reader) + } else { + reader.UnreadByte() + item, err = parseKeyValue(reader) + } + + if err != nil { + break + } + + if item == nil { + continue + } + + items = append(items, *item) + } + + if err == io.EOF { + err = nil + } + + return +} + +func parseComment(reader io.ByteScanner) (item *sops.TreeItem, err error) { + var builder strings.Builder + var whitespace bytes.Buffer + + for { + var b byte + b, err = reader.ReadByte() + + if err != nil { + break + } + + if b == '\n' { + break + } + + if isWhitespace(b) { + whitespace.WriteByte(b) + continue + } + + if builder.Len() == 0 { + whitespace.Reset() + } + + _, err = io.Copy(&builder, &whitespace) + + if err != nil { + break + } + + builder.WriteByte(b) + } + + if builder.Len() == 0 { + return + } + + item = &sops.TreeItem{Key: sops.Comment{builder.String()}, Value: nil} + return +} + +func parseKeyValue(reader io.ByteScanner) (item *sops.TreeItem, err error) { + var key, value string + + key, err = parseKey(reader) + if err != nil { + return + } + + value, err = parseValue(reader) + if err != nil { + return + } + + item = &sops.TreeItem{Key: key, Value: value} + return +} + +func parseKey(reader io.ByteScanner) (key string, err error) { + var builder strings.Builder + + for { + var b byte + b, err = reader.ReadByte() + + if err != nil { + break + } + + if b == '=' { + break + } + + builder.WriteByte(b) + } + + key = builder.String() + + if !KeyRegexp.MatchString(key) { + err = fmt.Errorf("invalid dotenv key: %q", key) + } + + return +} + +func parseValue(reader io.ByteScanner) (value string, err error) { + var first byte + first, err = reader.ReadByte() + + if err != nil { + return + } + + if first == '\'' { + return parseSingleQuoted(reader) + } + + if first == '"' { + return parseDoubleQuoted(reader) + } + + reader.UnreadByte() + return parseUnquoted(reader) +} + +func parseSingleQuoted(reader io.ByteScanner) (value string, err error) { + var builder strings.Builder + escaping := false + + for { + var b byte + b, err = reader.ReadByte() + + if err != nil { + break + } + + if !escaping && b == '\'' { + break + } + + if !escaping && b == '\\' { + escaping = true + continue + } + + if escaping && b != '\'' { + builder.WriteByte('\\') + } + + escaping = false + builder.WriteByte(b) + } + + value = builder.String() + return +} + +func parseDoubleQuoted(reader io.ByteScanner) (value string, err error) { + var builder strings.Builder + escaping := false + + for { + var b byte + b, err = reader.ReadByte() + + if err != nil { + break + } + + if !escaping && b == '"' { + break + } + + if !escaping && b == '\\' { + escaping = true + continue + } + + if escaping && b == 'n' { + b = '\n' + } else if escaping && b != '"' { + builder.WriteByte('\\') + } + + escaping = false + builder.WriteByte(b) + } + + value = builder.String() + return +} + +func parseUnquoted(reader io.ByteScanner) (value string, err error) { + var builder strings.Builder + var whitespace bytes.Buffer + + for { + var b byte + b, err = reader.ReadByte() + + if err != nil { + break + } + + if b == '\n' { + break + } + + if isWhitespace(b) { + whitespace.WriteByte(b) + continue + } + + if builder.Len() == 0 { + whitespace.Reset() + } + + _, err = io.Copy(&builder, &whitespace) + + if err != nil { + break + } + + builder.WriteByte(b) + } + + value = builder.String() + return +} + +func isWhitespace(b byte) bool { + return b == ' ' || b == '\t' || b == '\r' || b == '\n' +} diff --git a/stores/dotenv/store.go b/stores/dotenv/store.go index 8add8a097..eae2d7c83 100644 --- a/stores/dotenv/store.go +++ b/stores/dotenv/store.go @@ -63,30 +63,11 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { // sops runtime object func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { var branches sops.TreeBranches - var branch sops.TreeBranch - - for _, line := range bytes.Split(in, []byte("\n")) { - if len(line) == 0 { - continue - } - if line[0] == '#' { - branch = append(branch, sops.TreeItem{ - Key: sops.Comment{string(line[1:])}, - Value: nil, - }) - } else { - pos := bytes.Index(line, []byte("=")) - if pos == -1 { - return nil, fmt.Errorf("invalid dotenv input line: %s", line) - } - branch = append(branch, sops.TreeItem{ - Key: string(line[:pos]), - Value: strings.Replace(string(line[pos+1:]), "\\n", "\n", -1), - }) - } + items, err := parse(in) + if err != nil { + return nil, err } - - branches = append(branches, branch) + branches = append(branches, items) return branches, nil } @@ -117,10 +98,10 @@ func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { } var line string if comment, ok := item.Key.(sops.Comment); ok { - line = fmt.Sprintf("#%s\n", comment.Value) + line = fmt.Sprintf("# %s\n", comment.Value) } else { - value := strings.Replace(item.Value.(string), "\n", "\\n", -1) - line = fmt.Sprintf("%s=%s\n", item.Key, value) + value := strings.Replace(item.Value.(string), `'`, `\'`, -1) + line = fmt.Sprintf("%s='%s'\n", item.Key, value) } buffer.WriteString(line) } diff --git a/stores/dotenv/store_test.go b/stores/dotenv/store_test.go index f4bd2cc85..4b76cbd3c 100644 --- a/stores/dotenv/store_test.go +++ b/stores/dotenv/store_test.go @@ -8,49 +8,83 @@ import ( "go.mozilla.org/sops/v3" ) -var PLAIN = []byte(strings.TrimLeft(` -VAR1=val1 -VAR2=val2 -#comment -VAR3_unencrypted=val3 -VAR4=val4\nval4 +var ORIGINAL_PLAIN = []byte(strings.TrimLeft(` +#Comment +# Trimmed comment +UNQUOTED=value +UNQUOTED_ESCAPED_NEWLINE=escaped\nnewline +UNQUOTED_WHITESPACE= trimmed whitespace +SINGLEQUOTED='value' +SINGLEQUOTED_NEWLINE='real +newline' +SINGLEQUOTED_ESCAPED_NEWLINE='escaped\nnewline' +SINGLEQUOTED_ESCAPED_QUOTE='escaped\'quote' +SINGLEQUOTED_WHITESPACE=' untrimmed whitespace ' +DOUBLEQUOTED="value" +DOUBLEQUOTED_NEWLINE="real +newline" +DOUBLEQUOTED_ESCAPED_NEWLINE="real\nnewline" +DOUBLEQUOTED_ESCAPED_QUOTE="escaped\"quote" +DOUBLEQUOTED_WHITESPACE=" untrimmed whitespace " +`, "\n")) + +var EMITTED_PLAIN = []byte(strings.TrimLeft(` +# Comment +# Trimmed comment +UNQUOTED='value' +UNQUOTED_ESCAPED_NEWLINE='escaped\nnewline' +UNQUOTED_WHITESPACE='trimmed whitespace' +SINGLEQUOTED='value' +SINGLEQUOTED_NEWLINE='real +newline' +SINGLEQUOTED_ESCAPED_NEWLINE='escaped\nnewline' +SINGLEQUOTED_ESCAPED_QUOTE='escaped\'quote' +SINGLEQUOTED_WHITESPACE=' untrimmed whitespace ' +DOUBLEQUOTED='value' +DOUBLEQUOTED_NEWLINE='real +newline' +DOUBLEQUOTED_ESCAPED_NEWLINE='real +newline' +DOUBLEQUOTED_ESCAPED_QUOTE='escaped"quote' +DOUBLEQUOTED_WHITESPACE=' untrimmed whitespace ' `, "\n")) var BRANCH = sops.TreeBranch{ - sops.TreeItem{ - Key: "VAR1", - Value: "val1", - }, - sops.TreeItem{ - Key: "VAR2", - Value: "val2", - }, - sops.TreeItem{ - Key: sops.Comment{"comment"}, - Value: nil, - }, - sops.TreeItem{ - Key: "VAR3_unencrypted", - Value: "val3", - }, - sops.TreeItem{ - Key: "VAR4", - Value: "val4\nval4", - }, + sops.TreeItem{Key: sops.Comment{"Comment"}, Value: nil}, + sops.TreeItem{Key: sops.Comment{"Trimmed comment"}, Value: nil}, + sops.TreeItem{Key: "UNQUOTED", Value: "value"}, + sops.TreeItem{Key: "UNQUOTED_ESCAPED_NEWLINE", Value: "escaped\\nnewline"}, + sops.TreeItem{Key: "UNQUOTED_WHITESPACE", Value: "trimmed whitespace"}, + sops.TreeItem{Key: "SINGLEQUOTED", Value: "value"}, + sops.TreeItem{Key: "SINGLEQUOTED_NEWLINE", Value: "real\nnewline"}, + sops.TreeItem{Key: "SINGLEQUOTED_ESCAPED_NEWLINE", Value: "escaped\\nnewline"}, + sops.TreeItem{Key: "SINGLEQUOTED_ESCAPED_QUOTE", Value: "escaped'quote"}, + sops.TreeItem{Key: "SINGLEQUOTED_WHITESPACE", Value: " untrimmed whitespace "}, + sops.TreeItem{Key: "DOUBLEQUOTED", Value: "value"}, + sops.TreeItem{Key: "DOUBLEQUOTED_NEWLINE", Value: "real\nnewline"}, + sops.TreeItem{Key: "DOUBLEQUOTED_ESCAPED_NEWLINE", Value: "real\nnewline"}, + sops.TreeItem{Key: "DOUBLEQUOTED_ESCAPED_QUOTE", Value: "escaped\"quote"}, + sops.TreeItem{Key: "DOUBLEQUOTED_WHITESPACE", Value: " untrimmed whitespace "}, } func TestLoadPlainFile(t *testing.T) { - branches, err := (&Store{}).LoadPlainFile(PLAIN) + branches, err := (&Store{}).LoadPlainFile(ORIGINAL_PLAIN) assert.Nil(t, err) assert.Equal(t, BRANCH, branches[0]) } + +func TestInvalidKeyError(t *testing.T) { + _, err := (&Store{}).LoadPlainFile([]byte("INVALID KEY=irrelevant value")) + assert.Equal(t, err.Error(), "invalid dotenv key: \"INVALID KEY\"") +} + func TestEmitPlainFile(t *testing.T) { branches := sops.TreeBranches{ BRANCH, } bytes, err := (&Store{}).EmitPlainFile(branches) assert.Nil(t, err) - assert.Equal(t, PLAIN, bytes) + assert.Equal(t, EMITTED_PLAIN, bytes) } func TestEmitValueString(t *testing.T) {