Merge remote-tracking branch 'origin/develop' into sops-age-key-env

README.rst

@@ -189,13 +189,13 @@ the ``--age`` option or the **SOPS_AGE_RECIPIENTS** environment variable:
 
 .. code:: bash
 
-   $ sops --age age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw test.yaml > test.enc.yaml
+   $ sops --encrypt --age age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw test.yaml > test.enc.yaml
 
 When decrypting a file with the corresponding identity, sops will look for a
 text file name ``keys.txt`` located in a ``sops`` subdirectory of your user
-configuration directory. On Linux, this would be ``$XDG_CONFIG_HOME/sops/keys.txt``.
-On macOS, this would be ``$HOME/Library/Application Support/sops/keys.txt``. On
-Windows, this would be ``%AppData%\sops\keys.txt``. You can specify the location
+configuration directory. On Linux, this would be ``$XDG_CONFIG_HOME/sops/age/keys.txt``.
+On macOS, this would be ``$HOME/Library/Application Support/sops/age/keys.txt``. On
+Windows, this would be ``%AppData%\sops\age\keys.txt``. You can specify the location
 of this file manually by setting the environment variable **SOPS_AGE_KEY_FILE**.
 Alternatively you can provide the the key(s) directly by setting the **SOPS_AGE_KEY**
 environment variable.
@@ -652,7 +652,7 @@ and its KMS and PGP keys are used to encrypt the file. It should be noted that
 the looking up of ``.sops.yaml`` is from the working directory (CWD) instead of
 the directory of the encrypting file (see `Issue 242 <https://github.com/mozilla/sops/issues/242>`_).
 
-The path_regex checks the full path of the encrypting file. Here is another example:
+The path_regex checks the path of the encrypting file relative to the .sops.yaml config file. Here is another example:
 
 * files located under directory **development** should use one set of KMS A
 * files located under directory **production** should use another set of KMS B

config/config.go

@@ -8,7 +8,9 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"regexp"
+	"strings"
 
 	"github.com/sirupsen/logrus"
 	"go.mozilla.org/sops/v3"
@@ -313,12 +315,20 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio
 	return config, nil
 }
 
-func parseCreationRuleForFile(conf *configFile, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) {
+func parseCreationRuleForFile(conf *configFile, confPath, filePath string, kmsEncryptionContext map[string]*string) (*Config, error) {
 	// If config file doesn't contain CreationRules (it's empty or only contains DestionationRules), assume it does not exist
 	if conf.CreationRules == nil {
 		return nil, nil
 	}
 
+	configDir, err := filepath.Abs(filepath.Dir(confPath))
+	if err != nil {
+		return nil, err
+	}
+
+	// compare file path relative to path of config file
+	filePath = strings.TrimPrefix(filePath, configDir + string(filepath.Separator))
+
 	var rule *creationRule
 
 	for _, r := range conf.CreationRules {
@@ -356,7 +366,8 @@ func LoadCreationRuleForFile(confPath string, filePath string, kmsEncryptionCont
 	if err != nil {
 		return nil, err
 	}
-	return parseCreationRuleForFile(conf, filePath, kmsEncryptionContext)
+
+	return parseCreationRuleForFile(conf, confPath, filePath, kmsEncryptionContext)
 }
 
 // LoadDestinationRuleForFile works the same as LoadCreationRuleForFile, but gets the "creation_rule" from the matching destination_rule's

config/config_test.go

@@ -75,6 +75,15 @@ creation_rules:
     hc_vault_uris: https://foz:443/v1/foz/keys/foz
 `)
 
+var sampleConfigWithAmbiguousPath = []byte(`
+creation_rules:
+  - path_regex: foo/*
+    kms: "1"
+    pgp: "2"
+    gcp_kms: "3"
+    hc_vault_uris: http://4:8200/v1/4/keys/4
+`)
+
 var sampleConfigWithGroups = []byte(`
 creation_rules:
   - path_regex: foobar*
@@ -299,12 +308,12 @@ func TestLoadConfigFileWithGroups(t *testing.T) {
 }
 
 func TestLoadConfigFileWithNoMatchingRules(t *testing.T) {
-	_, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithNoMatchingRules, t), "foobar2000", nil)
+	_, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithNoMatchingRules, t), "/conf/path", "foobar2000", nil)
 	assert.NotNil(t, err)
 }
 
 func TestLoadConfigFileWithInvalidComplicatedRegexp(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidComplicatedRegexp, t), "stage/prod/api.yml", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidComplicatedRegexp, t), "/conf/path", "stage/prod/api.yml", nil)
 	assert.Equal(t, "can not compile regexp: error parsing regexp: invalid escape sequence: `\\K`", err.Error())
 	assert.Nil(t, conf)
 }
@@ -315,58 +324,58 @@ func TestLoadConfigFileWithComplicatedRegexp(t *testing.T) {
 		"stage/dev/feature-foo.yml": "dev-feature",
 		"stage/dev/api.yml":         "dev",
 	} {
-		conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithComplicatedRegexp, t), filePath, nil)
+		conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithComplicatedRegexp, t), "/conf/path", filePath, nil)
 		assert.Nil(t, err)
 		assert.Equal(t, k, conf.KeyGroups[0][0].ToString())
 	}
 }
 
 func TestLoadEmptyConfigFile(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleEmptyConfig, t), "foobar2000", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleEmptyConfig, t), "/conf/path", "foobar2000", nil)
 	assert.Nil(t, conf)
 	assert.Nil(t, err)
 }
 
 func TestLoadConfigFileWithEmptyCreationRules(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithEmptyCreationRules, t), "foobar2000", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithEmptyCreationRules, t), "/conf/path", "foobar2000", nil)
 	assert.Nil(t, conf)
 	assert.Nil(t, err)
 }
 
 func TestLoadConfigFileWithOnlyDestinationRules(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithOnlyDestinationRules, t), "foobar2000", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithOnlyDestinationRules, t), "/conf/path", "foobar2000", nil)
 	assert.Nil(t, conf)
 	assert.Nil(t, err)
 }
 
 func TestKeyGroupsForFile(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "foobar2000", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "/conf/path", "foobar2000", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "2", conf.KeyGroups[0][0].ToString())
 	assert.Equal(t, "1", conf.KeyGroups[0][1].ToString())
-	conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "whatever", nil)
+	conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "/conf/path", "whatever", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString())
 	assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString())
 }
 
 func TestKeyGroupsForFileWithPath(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "foo/bar2000", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "/conf/path", "foo/bar2000", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "2", conf.KeyGroups[0][0].ToString())
 	assert.Equal(t, "1", conf.KeyGroups[0][1].ToString())
-	conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "somefilename.yml", nil)
+	conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfigWithPath, t), "/conf/path", "somefilename.yml", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "baggins", conf.KeyGroups[0][0].ToString())
 	assert.Equal(t, "bilbo", conf.KeyGroups[0][1].ToString())
-	conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "whatever", nil)
+	conf, err = parseCreationRuleForFile(parseConfigFile(sampleConfig, t), "/conf/path", "whatever", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString())
 	assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString())
 }
 
 func TestKeyGroupsForFileWithGroups(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithGroups, t), "whatever", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithGroups, t), "/conf/path", "whatever", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "bar", conf.KeyGroups[0][0].ToString())
 	assert.Equal(t, "foo", conf.KeyGroups[0][1].ToString())
@@ -375,31 +384,39 @@ func TestKeyGroupsForFileWithGroups(t *testing.T) {
 }
 
 func TestLoadConfigFileWithUnencryptedSuffix(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "foobar", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "/conf/path", "foobar", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "_unencrypted", conf.UnencryptedSuffix)
 }
 
 func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "barfoo", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithSuffixParameters, t), "/conf/path", "barfoo", nil)
 	assert.Nil(t, err)
 	assert.Equal(t, "_enc", conf.EncryptedSuffix)
 }
 
 func TestLoadConfigFileWithUnencryptedRegex(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "/conf/path", "barbar", nil)
 	assert.Equal(t, nil, err)
 	assert.Equal(t, "^dec:", conf.UnencryptedRegex)
 }
 
 func TestLoadConfigFileWithEncryptedRegex(t *testing.T) {
-	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil)
+	conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "/conf/path", "barbar", nil)
 	assert.Equal(t, nil, err)
 	assert.Equal(t, "^enc:", conf.EncryptedRegex)
 }
 
 func TestLoadConfigFileWithInvalidParameters(t *testing.T) {
-	_, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidParameters, t), "foobar", nil)
+	_, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithInvalidParameters, t), "/conf/path", "foobar", nil)
+	assert.NotNil(t, err)
+}
+
+func TestLoadConfigFileWithAmbiguousPath(t *testing.T) {
+	config := parseConfigFile(sampleConfigWithAmbiguousPath, t)
+	_, err := parseCreationRuleForFile(config, "/foo/config", "/foo/foo/bar", nil)
+	assert.Nil(t, err)
+	_, err = parseCreationRuleForFile(config, "/foo/config", "/foo/fuu/bar", nil)
 	assert.NotNil(t, err)
 }
 

pgp/keysource_test.go

@@ -44,7 +44,10 @@ func TestPGPKeySourceFromString(t *testing.T) {
 }
 
 func TestRetrievePGPKey(t *testing.T) {
-	fingerprint := "FBC7B9E2A4F9289AC0C1D4843D16CEE4A27381B4"
+	// Requires a key available in https://keys.openpgp.org/ *with identity information* (that is, an email address).
+	// See https://keys.openpgp.org/about/faq#verify-multiple for details about identity information.
+	// We use the key of release@mozilla.com for here.
+	fingerprint := "14F26682D0916CDD81E37B6D61B7B526D98F0353"
 	_, err := getKeyFromKeyServer(fingerprint)
 	assert.NoError(t, err)
 }

stores/yaml/store.go

@@ -70,7 +70,7 @@ func (store Store) nodeToTreeValue(node *yaml.Node, commentsWereHandled bool) (i
 		return result, nil
 	case yaml.MappingNode:
 		branch := make(sops.TreeBranch, 0)
-		return store.appendYamlNodeToTreeBranch(node, branch, false)
+		return store.appendYamlNodeToTreeBranch(node, branch, commentsWereHandled)
 	case yaml.ScalarNode:
 		var result interface{}
 		node.Decode(&result)

stores/yaml/store_test.go

@@ -91,6 +91,26 @@ var COMMENT_5 = []byte(`# foo
 key: value
 `)
 
+// The following is a regression test for https://github.com/mozilla/sops/issues/865
+var COMMENT_6 = []byte(`a:
+    - a
+    # I no longer get duplicated
+    - {}
+`)
+
+var COMMENT_6_BRANCHES = sops.TreeBranches{
+	sops.TreeBranch{
+		sops.TreeItem{
+			Key:   "a",
+			Value: []interface{}{
+				"a",
+				sops.Comment{" I no longer get duplicated"},
+				sops.TreeBranch{},
+			},
+		},
+	},
+}
+
 func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) {
 	data := []byte(`hello: 2`)
 	_, err := (&Store{}).LoadEncryptedFile(data)
@@ -178,6 +198,16 @@ func TestEmpty2(t *testing.T) {
 }
 */
 
+func TestComment6(t *testing.T) {
+	branches, err := (&Store{}).LoadPlainFile(COMMENT_6)
+	assert.Nil(t, err)
+	assert.Equal(t, COMMENT_6_BRANCHES, branches)
+	bytes, err := (&Store{}).EmitPlainFile(branches)
+	assert.Nil(t, err)
+	assert.Equal(t, string(COMMENT_6), string(bytes))
+	assert.Equal(t, COMMENT_6, bytes)
+}
+
 func TestEmitValue(t *testing.T) {
 	// First iteration: load and store
 	bytes, err := (&Store{}).EmitValue(BRANCHES[0])