From 20aa308894848552605818d89f7c3a783fd34dc9 Mon Sep 17 00:00:00 2001 From: renehernandez Date: Thu, 6 Aug 2020 16:35:10 -0400 Subject: [PATCH 1/3] Add support for --unencrypted-regex --- README.rst | 13 ++++++++++- cmd/sops/edit.go | 4 +++- cmd/sops/encrypt.go | 2 ++ cmd/sops/main.go | 21 +++++++++++++---- config/config.go | 3 +++ config/config_test.go | 7 ++++++ sops.go | 29 +++++++++++++++++++---- sops_test.go | 53 +++++++++++++++++++++++++++++++++++++++++++ stores/stores.go | 8 ++++++- 9 files changed, 129 insertions(+), 11 deletions(-) diff --git a/README.rst b/README.rst index 864e87240..7f2405dac 100644 --- a/README.rst +++ b/README.rst @@ -1426,9 +1426,20 @@ will encrypt the values under the ``data`` and ``stringData`` keys in a YAML fil containing kubernetes secrets. It will not encrypt other values that help you to navigate the file, like ``metadata`` which contains the secrets' names. +Conversely, you can opt in to only left certain keys without encrypting by using the +``--unencrypted-regex`` option, which will leave the values unencrypted of those keys +that match the supplied regular expression. For example, this command: + +.. code:: bash + + $ sops --encrypt --unencrypted-regex '^(description|metadata)$' k8s-secrets.yaml + +will not encrypt the values under the ``description`` and ``metadata`` keys in a YAML file +containing kubernetes secrets, while encrypting everything else. + You can also specify these options in the ``.sops.yaml`` config file. -Note: these three options ``--unencrypted-suffix``, ``--encrypted-suffix``, and ``--encrypted-regex`` are +Note: these fourth options ``--unencrypted-suffix``, ``--encrypted-suffix``, ``--encrypted-regex`` and ``--unencrypted-regex`` are mutually exclusive and cannot all be used in the same file. Encryption Protocol diff --git a/cmd/sops/edit.go b/cmd/sops/edit.go index d137f5b22..2741319df 100644 --- a/cmd/sops/edit.go +++ b/cmd/sops/edit.go @@ -37,6 +37,7 @@ type editExampleOpts struct { editOpts UnencryptedSuffix string EncryptedSuffix string + UnencryptedRegex string EncryptedRegex string KeyGroups []sops.KeyGroup GroupThreshold int @@ -66,6 +67,7 @@ func editExample(opts editExampleOpts) ([]byte, error) { KeyGroups: opts.KeyGroups, UnencryptedSuffix: opts.UnencryptedSuffix, EncryptedSuffix: opts.EncryptedSuffix, + UnencryptedRegex: opts.UnencryptedRegex, EncryptedRegex: opts.EncryptedRegex, Version: version.Version, ShamirThreshold: opts.GroupThreshold, @@ -132,7 +134,7 @@ func editTree(opts editOpts, tree *sops.Tree, dataKey []byte) ([]byte, error) { if err != nil { return nil, common.NewExitError(fmt.Sprintf("Could not write output file: %s", err), codes.CouldNotWriteOutputFile) } - + // Close temporary file, since Windows won't delete the file unless it's closed beforehand defer tmpfile.Close() diff --git a/cmd/sops/encrypt.go b/cmd/sops/encrypt.go index 37dd642e6..1aa09eeba 100644 --- a/cmd/sops/encrypt.go +++ b/cmd/sops/encrypt.go @@ -22,6 +22,7 @@ type encryptOpts struct { KeyServices []keyservice.KeyServiceClient UnencryptedSuffix string EncryptedSuffix string + UnencryptedRegex string EncryptedRegex string KeyGroups []sops.KeyGroup GroupThreshold int @@ -77,6 +78,7 @@ func encrypt(opts encryptOpts) (encryptedFile []byte, err error) { KeyGroups: opts.KeyGroups, UnencryptedSuffix: opts.UnencryptedSuffix, EncryptedSuffix: opts.EncryptedSuffix, + UnencryptedRegex: opts.UnencryptedRegex, EncryptedRegex: opts.EncryptedRegex, Version: version.Version, ShamirThreshold: opts.GroupThreshold, diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 6a550d2cd..d1d72eb0c 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -185,9 +185,9 @@ func main() { Usage: "the user to run the command as", }, cli.StringFlag{ - Name: "input-type", - Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", - }, + Name: "input-type", + Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the file's extension to determine the type", + }, cli.StringFlag{ Name: "output-type", Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format", @@ -625,6 +625,10 @@ func main() { Name: "encrypted-suffix", Usage: "override the encrypted key suffix. When empty, all keys will be encrypted, unless otherwise marked with unencrypted-suffix.", }, + cli.StringFlag{ + Name: "unencrypted-regex", + Usage: "set the unencrypted key suffix. When specified, only keys matching the regex will be left unencrypted.", + }, cli.StringFlag{ Name: "encrypted-regex", Usage: "set the encrypted key suffix. When specified, only keys matching the regex will be encrypted.", @@ -682,6 +686,7 @@ func main() { unencryptedSuffix := c.String("unencrypted-suffix") encryptedSuffix := c.String("encrypted-suffix") encryptedRegex := c.String("encrypted-regex") + unencryptedRegex := c.String("unencrypted-regex") conf, err := loadConfig(c, fileName, nil) if err != nil { return toExitError(err) @@ -697,6 +702,9 @@ func main() { if encryptedRegex == "" { encryptedRegex = conf.EncryptedRegex } + if unencryptedRegex == "" { + unencryptedRegex = conf.UnencryptedRegex + } } cryptRuleCount := 0 @@ -709,9 +717,12 @@ func main() { if encryptedRegex != "" { cryptRuleCount++ } + if unencryptedRegex != "" { + cryptRuleCount++ + } if cryptRuleCount > 1 { - return common.NewExitError("Error: cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex in the same file", codes.ErrorConflictingParameters) + return common.NewExitError("Error: cannot use more than one of encrypted_suffix, unencrypted_suffix, encrypted_regex or unencrypted_regex in the same file", codes.ErrorConflictingParameters) } // only supply the default UnencryptedSuffix when EncryptedSuffix and EncryptedRegex are not provided @@ -742,6 +753,7 @@ func main() { Cipher: aes.NewCipher(), UnencryptedSuffix: unencryptedSuffix, EncryptedSuffix: encryptedSuffix, + UnencryptedRegex: unencryptedRegex, EncryptedRegex: encryptedRegex, KeyServices: svcs, KeyGroups: groups, @@ -879,6 +891,7 @@ func main() { editOpts: opts, UnencryptedSuffix: unencryptedSuffix, EncryptedSuffix: encryptedSuffix, + UnencryptedRegex: unencryptedRegex, EncryptedRegex: encryptedRegex, KeyGroups: groups, GroupThreshold: threshold, diff --git a/config/config.go b/config/config.go index 1ae6d8812..67a06cdda 100644 --- a/config/config.go +++ b/config/config.go @@ -117,6 +117,7 @@ type creationRule struct { ShamirThreshold int `yaml:"shamir_threshold"` UnencryptedSuffix string `yaml:"unencrypted_suffix"` EncryptedSuffix string `yaml:"encrypted_suffix"` + UnencryptedRegex string `yaml:"unencrypted_regex"` EncryptedRegex string `yaml:"encrypted_regex"` } @@ -135,6 +136,7 @@ type Config struct { ShamirThreshold int UnencryptedSuffix string EncryptedSuffix string + UnencryptedRegex string EncryptedRegex string Destination publish.Destination OmitExtensions bool @@ -235,6 +237,7 @@ func configFromRule(rule *creationRule, kmsEncryptionContext map[string]*string) ShamirThreshold: rule.ShamirThreshold, UnencryptedSuffix: rule.UnencryptedSuffix, EncryptedSuffix: rule.EncryptedSuffix, + UnencryptedRegex: rule.UnencryptedRegex, EncryptedRegex: rule.EncryptedRegex, }, nil } diff --git a/config/config_test.go b/config/config_test.go index 9f8710d48..41d9e854f 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -137,6 +137,7 @@ creation_rules: kms: "1" pgp: "2" encrypted_regex: "^enc:" + unencrypted_regex: "^dec:" `) var sampleConfigWithInvalidParameters = []byte(` @@ -349,6 +350,12 @@ func TestLoadConfigFileWithEncryptedSuffix(t *testing.T) { assert.Equal(t, "_enc", conf.EncryptedSuffix) } +func TestLoadConfigFileWithUnencryptedRegex(t *testing.T) { + conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil) + assert.Equal(t, nil, err) + assert.Equal(t, "^dec:", conf.UnencryptedRegex) +} + func TestLoadConfigFileWithEncryptedRegex(t *testing.T) { conf, err := parseCreationRuleForFile(parseConfigFile(sampleConfigWithRegexParameters, t), "barbar", nil) assert.Equal(t, nil, err) diff --git a/sops.go b/sops.go index d1d64ec8e..a41f4a2d1 100644 --- a/sops.go +++ b/sops.go @@ -287,9 +287,10 @@ func (branch TreeBranch) walkBranch(in TreeBranch, path []string, onLeaves func( // Encrypt walks over the tree and encrypts all values with the provided cipher, // except those whose key ends with the UnencryptedSuffix specified on the // Metadata struct, those not ending with EncryptedSuffix, if EncryptedSuffix -// is provided (by default it is not), or those not matching EncryptedRegex, -// if EncryptedRegex is provided (by default it is not). If encryption is -// successful, it returns the MAC for the encrypted tree. +// is provided (by default it is not), those not matching EncryptedRegex, +// if EncryptedRegex is provided (by default it is not) or those matching +// UnencryptedRegex, if UnencryptedRegex is provided (by default it is not). +// If encryption is successful, it returns the MAC for the encrypted tree. func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) { audit.SubmitEvent(audit.EncryptEvent{ File: tree.FilePath, @@ -323,6 +324,15 @@ func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) { } } } + if tree.Metadata.UnencryptedRegex != "" { + for _, p := range path { + matched, _ := regexp.Match(tree.Metadata.UnencryptedRegex, []byte(p)) + if matched { + encrypted = false + break + } + } + } if tree.Metadata.EncryptedRegex != "" { encrypted = false for _, p := range path { @@ -358,7 +368,8 @@ func (tree Tree) Encrypt(key []byte, cipher Cipher) (string, error) { // Decrypt walks over the tree and decrypts all values with the provided cipher, // except those whose key ends with the UnencryptedSuffix specified on the Metadata struct, // those not ending with EncryptedSuffix, if EncryptedSuffix is provided (by default it is not), -// or those not matching EncryptedRegex, if EncryptedRegex is provided (by default it is not). +// those not matching EncryptedRegex, if EncryptedRegex is provided (by default it is not), +// or those matching UnencryptedRegex, if UnencryptedRegex is provided (by default it is not). // If decryption is successful, it returns the MAC for the decrypted tree. func (tree Tree) Decrypt(key []byte, cipher Cipher) (string, error) { log.Debug("Decrypting tree") @@ -386,6 +397,15 @@ func (tree Tree) Decrypt(key []byte, cipher Cipher) (string, error) { } } } + if tree.Metadata.UnencryptedRegex != "" { + for _, p := range path { + matched, _ := regexp.Match(tree.Metadata.UnencryptedRegex, []byte(p)) + if matched { + encrypted = false + break + } + } + } if tree.Metadata.EncryptedRegex != "" { encrypted = false for _, p := range path { @@ -466,6 +486,7 @@ type Metadata struct { LastModified time.Time UnencryptedSuffix string EncryptedSuffix string + UnencryptedRegex string EncryptedRegex string MessageAuthenticationCode string Version string diff --git a/sops_test.go b/sops_test.go index d1d7fbc88..1386bbcc6 100644 --- a/sops_test.go +++ b/sops_test.go @@ -189,6 +189,59 @@ func TestEncryptedRegex(t *testing.T) { } } +func TestUnencryptedRegex(t *testing.T) { + branches := TreeBranches{ + TreeBranch{ + TreeItem{ + Key: "dec:foo", + Value: "bar", + }, + TreeItem{ + Key: "dec:bar", + Value: TreeBranch{ + TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + }, + } + tree := Tree{Branches: branches, Metadata: Metadata{UnencryptedRegex: "^dec:"}} + expected := TreeBranch{ + TreeItem{ + Key: "dec:foo", + Value: "bar", + }, + TreeItem{ + Key: "dec:bar", + Value: TreeBranch{ + TreeItem{ + Key: "foo", + Value: "bar", + }, + }, + }, + } + cipher := reverseCipher{} + + _, err := tree.Encrypt(bytes.Repeat([]byte("f"), 32), cipher) + if err != nil { + t.Errorf("Encrypting the tree failed: %s", err) + } + // expected[1].Value[] = "bar" + if !reflect.DeepEqual(tree.Branches[0], expected) { + t.Errorf("Trees don't match: \ngot \t\t%+v,\n expected \t\t%+v", tree.Branches[0], expected) + } + _, err = tree.Decrypt(bytes.Repeat([]byte("f"), 32), cipher) + if err != nil { + t.Errorf("Decrypting the tree failed: %s", err) + } + if !reflect.DeepEqual(tree.Branches[0], expected) { + t.Errorf("Trees don't match: \ngot\t\t\t%+v,\nexpected\t\t%+v", tree.Branches[0], expected) + } +} + type MockCipher struct{} func (m MockCipher) Encrypt(value interface{}, key []byte, path string) (string, error) { diff --git a/stores/stores.go b/stores/stores.go index 323a91673..236940fb2 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -47,6 +47,7 @@ type Metadata struct { PGPKeys []pgpkey `yaml:"pgp" json:"pgp"` UnencryptedSuffix string `yaml:"unencrypted_suffix,omitempty" json:"unencrypted_suffix,omitempty"` EncryptedSuffix string `yaml:"encrypted_suffix,omitempty" json:"encrypted_suffix,omitempty"` + UnencryptedRegex string `yaml:"unencrypted_regex,omitempty" json:"unencrypted_regex,omitempty"` EncryptedRegex string `yaml:"encrypted_regex,omitempty" json:"encrypted_regex,omitempty"` Version string `yaml:"version" json:"version"` } @@ -102,6 +103,7 @@ func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { m.LastModified = sopsMetadata.LastModified.Format(time.RFC3339) m.UnencryptedSuffix = sopsMetadata.UnencryptedSuffix m.EncryptedSuffix = sopsMetadata.EncryptedSuffix + m.UnencryptedRegex = sopsMetadata.UnencryptedRegex m.EncryptedRegex = sopsMetadata.EncryptedRegex m.MessageAuthenticationCode = sopsMetadata.MessageAuthenticationCode m.Version = sopsMetadata.Version @@ -222,12 +224,15 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) { if m.EncryptedSuffix != "" { cryptRuleCount++ } + if m.UnencryptedRegex != "" { + cryptRuleCount++ + } if m.EncryptedRegex != "" { cryptRuleCount++ } if cryptRuleCount > 1 { - return sops.Metadata{}, fmt.Errorf("Cannot use more than one of encrypted_suffix, unencrypted_suffix, or encrypted_regex in the same file") + return sops.Metadata{}, fmt.Errorf("Cannot use more than one of encrypted_suffix, unencrypted_suffix, encrypted_regex or unencrypted_regex in the same file") } if cryptRuleCount == 0 { @@ -240,6 +245,7 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) { MessageAuthenticationCode: m.MessageAuthenticationCode, UnencryptedSuffix: m.UnencryptedSuffix, EncryptedSuffix: m.EncryptedSuffix, + UnencryptedRegex: m.UnencryptedRegex, EncryptedRegex: m.EncryptedRegex, LastModified: lastModified, }, nil From 94bd98e8bce683b1ab7d597c0ca2becd2bb2ce94 Mon Sep 17 00:00:00 2001 From: renehernandez Date: Thu, 6 Aug 2020 16:37:48 -0400 Subject: [PATCH 2/3] Fix grammar mistake --- README.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.rst b/README.rst index 7f2405dac..64fe6315f 100644 --- a/README.rst +++ b/README.rst @@ -1439,7 +1439,7 @@ containing kubernetes secrets, while encrypting everything else. You can also specify these options in the ``.sops.yaml`` config file. -Note: these fourth options ``--unencrypted-suffix``, ``--encrypted-suffix``, ``--encrypted-regex`` and ``--unencrypted-regex`` are +Note: these four options ``--unencrypted-suffix``, ``--encrypted-suffix``, ``--encrypted-regex`` and ``--unencrypted-regex`` are mutually exclusive and cannot all be used in the same file. Encryption Protocol From 06af1b2b1dd2de02723c4f1e26e4db04218af118 Mon Sep 17 00:00:00 2001 From: renehernandez Date: Fri, 7 Aug 2020 10:13:12 -0400 Subject: [PATCH 3/3] Add gofmt'd files --- hcvault/keysource.go | 4 ++-- keyservice/keyservice.go | 4 ++-- keyservice/keyservice.pb.go | 2 +- stores/dotenv/store_test.go | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/hcvault/keysource.go b/hcvault/keysource.go index 4f4b5124c..4d436f0dd 100644 --- a/hcvault/keysource.go +++ b/hcvault/keysource.go @@ -29,7 +29,7 @@ func init() { type MasterKey struct { EncryptedKey string KeyName string - EnginePath string + EnginePath string VaultAddress string CreationDate time.Time } @@ -100,7 +100,7 @@ func getBackendAndKeyFromPath(fullPath string) (enginePath, keyName string, err func NewMasterKey(addess, enginePath, keyName string) *MasterKey { mk := &MasterKey{ VaultAddress: addess, - EnginePath: enginePath, + EnginePath: enginePath, KeyName: keyName, CreationDate: time.Now().UTC(), } diff --git a/keyservice/keyservice.go b/keyservice/keyservice.go index f67732ff8..f60e15822 100644 --- a/keyservice/keyservice.go +++ b/keyservice/keyservice.go @@ -39,8 +39,8 @@ func KeyFromMasterKey(mk keys.MasterKey) Key { KeyType: &Key_VaultKey{ VaultKey: &VaultKey{ VaultAddress: mk.VaultAddress, - EnginePath: mk.EnginePath, - KeyName: mk.KeyName, + EnginePath: mk.EnginePath, + KeyName: mk.KeyName, }, }, } diff --git a/keyservice/keyservice.pb.go b/keyservice/keyservice.pb.go index 433baabb3..c8177144b 100644 --- a/keyservice/keyservice.pb.go +++ b/keyservice/keyservice.pb.go @@ -293,7 +293,7 @@ func (m *GcpKmsKey) GetResourceId() string { type VaultKey struct { VaultAddress string `protobuf:"bytes,1,opt,name=vault_address,json=vaultAddress,proto3" json:"vault_address,omitempty"` - EnginePath string `protobuf:"bytes,2,opt,name=engine_path,json=enginePath,proto3" json:"engine_path,omitempty"` + EnginePath string `protobuf:"bytes,2,opt,name=engine_path,json=enginePath,proto3" json:"engine_path,omitempty"` KeyName string `protobuf:"bytes,3,opt,name=key_name,json=keyName,proto3" json:"key_name,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` diff --git a/stores/dotenv/store_test.go b/stores/dotenv/store_test.go index 645eb65cf..f4bd2cc85 100644 --- a/stores/dotenv/store_test.go +++ b/stores/dotenv/store_test.go @@ -62,4 +62,4 @@ func TestEmitValueString(t *testing.T) { func TestEmitValueNonstring(t *testing.T) { _, err := (&Store{}).EmitValue(BRANCH) assert.NotNil(t, err) -} \ No newline at end of file +}