From 8a216e92605407b71af0fa39201564ce23469df7 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Thu, 9 Jan 2020 09:07:39 +0600 Subject: [PATCH 1/9] Add `-recurse` option for publish subcommand --- cmd/sops/main.go | 9 ++++--- cmd/sops/subcommand/publish/publish.go | 35 +++++++++++++++++++++----- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 3f453c8f4..81c412e3a 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -211,13 +211,17 @@ func main() { }, { Name: "publish", - Usage: "Publish sops file to a configured destination", + Usage: "Publish sops file or directory to a configured destination", ArgsUsage: `file`, Flags: append([]cli.Flag{ cli.BoolFlag{ Name: "yes, y", Usage: `pre-approve all changes and run non-interactively`, }, + cli.BoolFlag{ + Name: "recurse", + Usage: "If source path is directory, publish all its content recursively", + }, cli.BoolFlag{ Name: "verbose", Usage: "Enable verbose logging output", @@ -235,14 +239,13 @@ func main() { return common.NewExitError("Error: no file specified", codes.NoFileSpecified) } fileName := c.Args()[0] - inputStore := inputStore(c, fileName) err = publishcmd.Run(publishcmd.Opts{ ConfigPath: configPath, InputPath: fileName, - InputStore: inputStore, Cipher: aes.NewCipher(), KeyServices: keyservices(c), Interactive: !c.Bool("yes"), + Recurse: c.Bool("recurse"), }) if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { return cliErr diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index d0d27bc07..855c5f2b0 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -33,6 +33,7 @@ type Opts struct { InputPath string KeyServices []keyservice.KeyServiceClient InputStore sops.Store + Recurse bool } // Run publish operation @@ -46,10 +47,26 @@ func Run(opts Opts) error { if err != nil { return err } - if info.IsDir() { + if info.IsDir() && !opts.Recurse { return fmt.Errorf("can't operate on a directory") + } else if info.IsDir() && opts.Recurse { + err = filepath.Walk(opts.InputPath, func(subPath string, info os.FileInfo, err error) error { + subAbsPath, _ := filepath.Abs(subPath) + if !info.IsDir() && subAbsPath != path { + subOpts := opts + subOpts.InputPath = subPath + return Run(subOpts) + } else { + return nil + } + }) + if err != nil { + return err + } + return nil } - _, fileName := filepath.Split(path) + opts.InputStore = common.DefaultStoreForPathOrFormat(path, filepath.Ext(path)) + destinationPath := opts.InputPath conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string)) if err != nil { @@ -146,22 +163,28 @@ func Run(opts Opts) error { if opts.Interactive { var response string for response != "y" && response != "n" { - fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(fileName)) + fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(destinationPath)) _, err := fmt.Scanln(&response) if err != nil { return err } } if response == "n" { - return errors.New("Publish canceled") + msg := fmt.Sprintf("Publication of %s canceled", path) + if opts.Recurse { + fmt.Println(msg) + return nil + } else { + return errors.New(msg) + } } } switch dest := conf.Destination.(type) { case *publish.S3Destination, *publish.GCSDestination: - err = dest.Upload(fileContents, fileName) + err = dest.Upload(fileContents, destinationPath) case *publish.VaultDestination: - err = dest.UploadUnencrypted(data, fileName) + err = dest.UploadUnencrypted(data, destinationPath) } if err != nil { From ef68940d2fe8f9500cc6cf5aa09fcb91e55ebb57 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Thu, 9 Jan 2020 09:18:51 +0600 Subject: [PATCH 2/9] Add `-omit-extensions` option for publish subcommand --- cmd/sops/main.go | 17 +++++++++++------ cmd/sops/subcommand/publish/publish.go | 22 ++++++++++++++-------- config/config.go | 3 +++ 3 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 81c412e3a..c676480eb 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -218,6 +218,10 @@ func main() { Name: "yes, y", Usage: `pre-approve all changes and run non-interactively`, }, + cli.BoolFlag{ + Name: "omit-extensions", + Usage: "Omit file extensions in destination path when publishing sops file to configured destinations", + }, cli.BoolFlag{ Name: "recurse", Usage: "If source path is directory, publish all its content recursively", @@ -240,12 +244,13 @@ func main() { } fileName := c.Args()[0] err = publishcmd.Run(publishcmd.Opts{ - ConfigPath: configPath, - InputPath: fileName, - Cipher: aes.NewCipher(), - KeyServices: keyservices(c), - Interactive: !c.Bool("yes"), - Recurse: c.Bool("recurse"), + ConfigPath: configPath, + InputPath: fileName, + Cipher: aes.NewCipher(), + KeyServices: keyservices(c), + Interactive: !c.Bool("yes"), + OmitExtensions: c.Bool("omit-extensions"), + Recurse: c.Bool("recurse"), }) if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { return cliErr diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index 855c5f2b0..6c22988ca 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/cmd/sops/codes" @@ -27,13 +28,14 @@ func init() { // Opts represents publish options and config type Opts struct { - Interactive bool - Cipher sops.Cipher - ConfigPath string - InputPath string - KeyServices []keyservice.KeyServiceClient - InputStore sops.Store - Recurse bool + Interactive bool + Cipher sops.Cipher + ConfigPath string + InputPath string + KeyServices []keyservice.KeyServiceClient + InputStore sops.Store + OmitExtensions bool + Recurse bool } // Run publish operation @@ -65,7 +67,8 @@ func Run(opts Opts) error { } return nil } - opts.InputStore = common.DefaultStoreForPathOrFormat(path, filepath.Ext(path)) + fileSuffix := filepath.Ext(path) + opts.InputStore = common.DefaultStoreForPathOrFormat(path, fileSuffix) destinationPath := opts.InputPath conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string)) @@ -75,6 +78,9 @@ func Run(opts Opts) error { if conf.Destination == nil { return errors.New("no destination configured for this file") } + if opts.OmitExtensions || conf.OmitExtensions { + destinationPath = strings.TrimSuffix(destinationPath, fileSuffix) + } // Check that this is a sops-encrypted file tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath) diff --git a/config/config.go b/config/config.go index 8e42c3a6b..62f48913d 100644 --- a/config/config.go +++ b/config/config.go @@ -100,6 +100,7 @@ type destinationRule struct { VaultKVMountName string `yaml:"vault_kv_mount_name"` VaultKVVersion int `yaml:"vault_kv_version"` RecreationRule creationRule `yaml:"recreation_rule,omitempty"` + OmitExtensions bool `yaml:"omit_extensions"` } type creationRule struct { @@ -133,6 +134,7 @@ type Config struct { EncryptedSuffix string EncryptedRegex string Destination publish.Destination + OmitExtensions bool } func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[string]*string) ([]sops.KeyGroup, error) { @@ -266,6 +268,7 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio return nil, err } config.Destination = dest + config.OmitExtensions = dRule.OmitExtensions return config, nil } From 10ef21c8b8243ab0efaab1839fac235dfbd9e2b5 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Thu, 9 Jan 2020 10:51:14 +0600 Subject: [PATCH 3/9] Skip publish to Vault if secret is up-to-date in order to avoid version increment --- publish/vault.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/publish/vault.go b/publish/vault.go index 01e0ddc0e..301e13bc3 100644 --- a/publish/vault.go +++ b/publish/vault.go @@ -4,6 +4,7 @@ import ( "fmt" "strings" + "github.com/google/go-cmp/cmp" vault "github.com/hashicorp/vault/api" ) @@ -65,6 +66,17 @@ func (vaultd *VaultDestination) UploadUnencrypted(data map[string]interface{}, f } } + existingSecret, err := client.Logical().Read(vaultd.secretsPath(fileName)) + if err != nil { + return err + } + if existingSecret != nil { + if cmp.Equal(data, existingSecret.Data["data"]) { + fmt.Printf("Secret in %s is already up-to-date.\n", vaultd.secretsPath(fileName)) + return nil + } + } + secretsData := make(map[string]interface{}) if vaultd.kvVersion == 1 { From 4254322d7edfd3a4b9c1247efb4cf479a656c544 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Thu, 9 Jan 2020 11:25:25 +0600 Subject: [PATCH 4/9] update docs --- README.rst | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/README.rst b/README.rst index 6d22fbc14..87a81679b 100644 --- a/README.rst +++ b/README.rst @@ -968,6 +968,7 @@ This command requires a ``.sops.yaml`` configuration file. Below is an example: vault_kv_mount_name: "secret/" # default vault_kv_version: 2 # default path_regex: vault/* + omit_extensions: true The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets``, all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, and the contents of all files under @@ -977,6 +978,11 @@ published to S3 and GCS, it will decrypt them and re-encrypt them using the You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml`` +To publish all files in selected directory recursively, you need to specify ``--recurse`` flag. + +If you don't want file extension to appear in destination secret path, use ``--omit-extensions`` +flag or same ``.sops.yaml`` option. + Publishing to Vault ******************* @@ -991,6 +997,9 @@ configuring the client. ``vault_kv_mount_name`` is used if your Vault KV is mounted somewhere other than ``secret/``. ``vault_kv_version`` supports ``1`` and ``2``, with ``2`` being the default. +If destination secret path already exists in Vault and contains same data as source file, it will +be skipped. + Below is an example of publishing to Vault (using token auth with a local dev instance of Vault). .. code:: bash From 3ab2d41c2f596e1fe5fa59c72a559514bcc91080 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Fri, 10 Jan 2020 09:33:03 +0600 Subject: [PATCH 5/9] fix filepath.Walk abuse; rename recursive flag; minor fixes --- README.rst | 8 ++--- cmd/sops/main.go | 46 ++++++++++++++++++-------- cmd/sops/subcommand/publish/publish.go | 38 ++------------------- 3 files changed, 39 insertions(+), 53 deletions(-) diff --git a/README.rst b/README.rst index 87a81679b..4c597a33f 100644 --- a/README.rst +++ b/README.rst @@ -978,10 +978,10 @@ published to S3 and GCS, it will decrypt them and re-encrypt them using the You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml`` -To publish all files in selected directory recursively, you need to specify ``--recurse`` flag. +To publish all files in selected directory recursively, you need to specify ``--recursive`` flag. If you don't want file extension to appear in destination secret path, use ``--omit-extensions`` -flag or same ``.sops.yaml`` option. +flag or ``omit_extensions: true`` in the destination rule in ``.sops.yaml``. Publishing to Vault ******************* @@ -997,8 +997,8 @@ configuring the client. ``vault_kv_mount_name`` is used if your Vault KV is mounted somewhere other than ``secret/``. ``vault_kv_version`` supports ``1`` and ``2``, with ``2`` being the default. -If destination secret path already exists in Vault and contains same data as source file, it will -be skipped. +If destination secret path already exists in Vault and contains same data as the source file, it +will be skipped. Below is an example of publishing to Vault (using token auth with a local dev instance of Vault). diff --git a/cmd/sops/main.go b/cmd/sops/main.go index c676480eb..83393af20 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -223,7 +223,7 @@ func main() { Usage: "Omit file extensions in destination path when publishing sops file to configured destinations", }, cli.BoolFlag{ - Name: "recurse", + Name: "recursive", Usage: "If source path is directory, publish all its content recursively", }, cli.BoolFlag{ @@ -242,20 +242,38 @@ func main() { if c.NArg() < 1 { return common.NewExitError("Error: no file specified", codes.NoFileSpecified) } - fileName := c.Args()[0] - err = publishcmd.Run(publishcmd.Opts{ - ConfigPath: configPath, - InputPath: fileName, - Cipher: aes.NewCipher(), - KeyServices: keyservices(c), - Interactive: !c.Bool("yes"), - OmitExtensions: c.Bool("omit-extensions"), - Recurse: c.Bool("recurse"), + path := c.Args()[0] + info, err := os.Stat(path) + if err != nil { + return err + } + if info.IsDir() && !c.Bool("recursive") { + return fmt.Errorf("can't operate on a directory without --recursive flag.") + } + err = filepath.Walk(path, func(subPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + err = publishcmd.Run(publishcmd.Opts{ + ConfigPath: configPath, + InputPath: subPath, + Cipher: aes.NewCipher(), + KeyServices: keyservices(c), + InputStore: inputStore(c, subPath), + Interactive: !c.Bool("yes"), + OmitExtensions: c.Bool("omit-extensions"), + }) + if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { + return cliErr + } else if err != nil { + return common.NewExitError(err, codes.ErrorGeneric) + } + } + return nil }) - if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { - return cliErr - } else if err != nil { - return common.NewExitError(err, codes.ErrorGeneric) + if err != nil { + return err } return nil }, diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index 6c22988ca..fa7dba8cf 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "io/ioutil" - "os" "path/filepath" "strings" @@ -35,7 +34,6 @@ type Opts struct { KeyServices []keyservice.KeyServiceClient InputStore sops.Store OmitExtensions bool - Recurse bool } // Run publish operation @@ -45,31 +43,6 @@ func Run(opts Opts) error { if err != nil { return err } - info, err := os.Stat(path) - if err != nil { - return err - } - if info.IsDir() && !opts.Recurse { - return fmt.Errorf("can't operate on a directory") - } else if info.IsDir() && opts.Recurse { - err = filepath.Walk(opts.InputPath, func(subPath string, info os.FileInfo, err error) error { - subAbsPath, _ := filepath.Abs(subPath) - if !info.IsDir() && subAbsPath != path { - subOpts := opts - subOpts.InputPath = subPath - return Run(subOpts) - } else { - return nil - } - }) - if err != nil { - return err - } - return nil - } - fileSuffix := filepath.Ext(path) - opts.InputStore = common.DefaultStoreForPathOrFormat(path, fileSuffix) - destinationPath := opts.InputPath conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string)) if err != nil { @@ -78,8 +51,9 @@ func Run(opts Opts) error { if conf.Destination == nil { return errors.New("no destination configured for this file") } + destinationPath := opts.InputPath if opts.OmitExtensions || conf.OmitExtensions { - destinationPath = strings.TrimSuffix(destinationPath, fileSuffix) + destinationPath = strings.TrimSuffix(destinationPath, filepath.Ext(path)) } // Check that this is a sops-encrypted file @@ -176,13 +150,7 @@ func Run(opts Opts) error { } } if response == "n" { - msg := fmt.Sprintf("Publication of %s canceled", path) - if opts.Recurse { - fmt.Println(msg) - return nil - } else { - return errors.New(msg) - } + return errors.New("Publish canceled") } } From 01b5fb627997bfc3956c2a0ac3fec85c63ac5f76 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Sat, 11 Jan 2020 10:47:20 +0600 Subject: [PATCH 6/9] Dont fail Vault publish with write-only access; improve vault publish logging --- publish/vault.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/publish/vault.go b/publish/vault.go index 301e13bc3..bfc5b3ba0 100644 --- a/publish/vault.go +++ b/publish/vault.go @@ -6,8 +6,17 @@ import ( "github.com/google/go-cmp/cmp" vault "github.com/hashicorp/vault/api" + "go.mozilla.org/sops/v3/logging" + + "github.com/sirupsen/logrus" ) +var log *logrus.Logger + +func init() { + log = logging.NewLogger("PUBLISH") +} + type VaultDestination struct { vaultAddress string vaultPath string @@ -66,15 +75,14 @@ func (vaultd *VaultDestination) UploadUnencrypted(data map[string]interface{}, f } } - existingSecret, err := client.Logical().Read(vaultd.secretsPath(fileName)) + secretsPath := vaultd.secretsPath(fileName) + existingSecret, err := client.Logical().Read(secretsPath) if err != nil { - return err + log.Warnf("Cannot check if destination secret already exists in %s. New version will be created even if the data has not been changed.", secretsPath) } - if existingSecret != nil { - if cmp.Equal(data, existingSecret.Data["data"]) { - fmt.Printf("Secret in %s is already up-to-date.\n", vaultd.secretsPath(fileName)) - return nil - } + if existingSecret != nil && cmp.Equal(data, existingSecret.Data["data"]) { + log.Infof("Secret in %s is already up-to-date.\n", secretsPath) + return nil } secretsData := make(map[string]interface{}) @@ -85,7 +93,7 @@ func (vaultd *VaultDestination) UploadUnencrypted(data map[string]interface{}, f secretsData["data"] = data } - _, err = client.Logical().Write(vaultd.secretsPath(fileName), secretsData) + _, err = client.Logical().Write(secretsPath, secretsData) if err != nil { return err } From 3ccc7e4067c47f14868a4ce956e50dde79555a68 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Mon, 13 Jan 2020 15:00:33 +0600 Subject: [PATCH 7/9] Fix destination path on single file publish --- cmd/sops/main.go | 1 + cmd/sops/subcommand/publish/publish.go | 17 +++++++++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 83393af20..6c138dc48 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -263,6 +263,7 @@ func main() { InputStore: inputStore(c, subPath), Interactive: !c.Bool("yes"), OmitExtensions: c.Bool("omit-extensions"), + Recursive: c.Bool("recursive"), }) if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { return cliErr diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index fa7dba8cf..408c50621 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -34,6 +34,7 @@ type Opts struct { KeyServices []keyservice.KeyServiceClient InputStore sops.Store OmitExtensions bool + Recursive bool } // Run publish operation @@ -51,7 +52,13 @@ func Run(opts Opts) error { if conf.Destination == nil { return errors.New("no destination configured for this file") } - destinationPath := opts.InputPath + + var destinationPath string + if opts.Recursive { + destinationPath = opts.InputPath + } else { + _, destinationPath = filepath.Split(path) + } if opts.OmitExtensions || conf.OmitExtensions { destinationPath = strings.TrimSuffix(destinationPath, filepath.Ext(path)) } @@ -150,7 +157,13 @@ func Run(opts Opts) error { } } if response == "n" { - return errors.New("Publish canceled") + msg := fmt.Sprintf("Publication of %s canceled", path) + if opts.Recursive { + fmt.Println(msg) + return nil + } else { + return errors.New(msg) + } } } From 3db9c71596adc05ca1d15f192b1a94d7f0762f77 Mon Sep 17 00:00:00 2001 From: Mikhail Morev Date: Tue, 14 Jan 2020 00:45:55 +0300 Subject: [PATCH 8/9] Update cmd/sops/main.go Co-Authored-By: AJ Bahnken <1144310+ajvb@users.noreply.github.com> --- cmd/sops/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 6c138dc48..61dc36e3b 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -224,7 +224,7 @@ func main() { }, cli.BoolFlag{ Name: "recursive", - Usage: "If source path is directory, publish all its content recursively", + Usage: "If the source path is a directory, publish all its content recursively", }, cli.BoolFlag{ Name: "verbose", From 0c6558b7f271275a67432f93f7877698a33cb8a9 Mon Sep 17 00:00:00 2001 From: MOREV Mikhail Date: Tue, 14 Jan 2020 15:36:26 +0300 Subject: [PATCH 9/9] Recursive publish - use relative paths --- cmd/sops/main.go | 1 + cmd/sops/subcommand/publish/publish.go | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 6c138dc48..1f298403b 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -264,6 +264,7 @@ func main() { Interactive: !c.Bool("yes"), OmitExtensions: c.Bool("omit-extensions"), Recursive: c.Bool("recursive"), + RootPath: path, }) if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { return cliErr diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index 408c50621..4f3804ca0 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -35,6 +35,7 @@ type Opts struct { InputStore sops.Store OmitExtensions bool Recursive bool + RootPath string } // Run publish operation @@ -55,7 +56,10 @@ func Run(opts Opts) error { var destinationPath string if opts.Recursive { - destinationPath = opts.InputPath + destinationPath, err = filepath.Rel(opts.RootPath, opts.InputPath) + if err != nil { + return err + } } else { _, destinationPath = filepath.Split(path) }