diff --git a/README.rst b/README.rst index 6d22fbc14..4c597a33f 100644 --- a/README.rst +++ b/README.rst @@ -968,6 +968,7 @@ This command requires a ``.sops.yaml`` configuration file. Below is an example: vault_kv_mount_name: "secret/" # default vault_kv_version: 2 # default path_regex: vault/* + omit_extensions: true The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets``, all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, and the contents of all files under @@ -977,6 +978,11 @@ published to S3 and GCS, it will decrypt them and re-encrypt them using the You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml`` +To publish all files in selected directory recursively, you need to specify ``--recursive`` flag. + +If you don't want file extension to appear in destination secret path, use ``--omit-extensions`` +flag or ``omit_extensions: true`` in the destination rule in ``.sops.yaml``. + Publishing to Vault ******************* @@ -991,6 +997,9 @@ configuring the client. ``vault_kv_mount_name`` is used if your Vault KV is mounted somewhere other than ``secret/``. ``vault_kv_version`` supports ``1`` and ``2``, with ``2`` being the default. +If destination secret path already exists in Vault and contains same data as the source file, it +will be skipped. + Below is an example of publishing to Vault (using token auth with a local dev instance of Vault). .. code:: bash diff --git a/cmd/sops/main.go b/cmd/sops/main.go index 3f453c8f4..7561dcecf 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -211,13 +211,21 @@ func main() { }, { Name: "publish", - Usage: "Publish sops file to a configured destination", + Usage: "Publish sops file or directory to a configured destination", ArgsUsage: `file`, Flags: append([]cli.Flag{ cli.BoolFlag{ Name: "yes, y", Usage: `pre-approve all changes and run non-interactively`, }, + cli.BoolFlag{ + Name: "omit-extensions", + Usage: "Omit file extensions in destination path when publishing sops file to configured destinations", + }, + cli.BoolFlag{ + Name: "recursive", + Usage: "If the source path is a directory, publish all its content recursively", + }, cli.BoolFlag{ Name: "verbose", Usage: "Enable verbose logging output", @@ -234,20 +242,40 @@ func main() { if c.NArg() < 1 { return common.NewExitError("Error: no file specified", codes.NoFileSpecified) } - fileName := c.Args()[0] - inputStore := inputStore(c, fileName) - err = publishcmd.Run(publishcmd.Opts{ - ConfigPath: configPath, - InputPath: fileName, - InputStore: inputStore, - Cipher: aes.NewCipher(), - KeyServices: keyservices(c), - Interactive: !c.Bool("yes"), + path := c.Args()[0] + info, err := os.Stat(path) + if err != nil { + return err + } + if info.IsDir() && !c.Bool("recursive") { + return fmt.Errorf("can't operate on a directory without --recursive flag.") + } + err = filepath.Walk(path, func(subPath string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if !info.IsDir() { + err = publishcmd.Run(publishcmd.Opts{ + ConfigPath: configPath, + InputPath: subPath, + Cipher: aes.NewCipher(), + KeyServices: keyservices(c), + InputStore: inputStore(c, subPath), + Interactive: !c.Bool("yes"), + OmitExtensions: c.Bool("omit-extensions"), + Recursive: c.Bool("recursive"), + RootPath: path, + }) + if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { + return cliErr + } else if err != nil { + return common.NewExitError(err, codes.ErrorGeneric) + } + } + return nil }) - if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil { - return cliErr - } else if err != nil { - return common.NewExitError(err, codes.ErrorGeneric) + if err != nil { + return err } return nil }, diff --git a/cmd/sops/subcommand/publish/publish.go b/cmd/sops/subcommand/publish/publish.go index d0d27bc07..4f3804ca0 100644 --- a/cmd/sops/subcommand/publish/publish.go +++ b/cmd/sops/subcommand/publish/publish.go @@ -4,8 +4,8 @@ import ( "errors" "fmt" "io/ioutil" - "os" "path/filepath" + "strings" "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/cmd/sops/codes" @@ -27,12 +27,15 @@ func init() { // Opts represents publish options and config type Opts struct { - Interactive bool - Cipher sops.Cipher - ConfigPath string - InputPath string - KeyServices []keyservice.KeyServiceClient - InputStore sops.Store + Interactive bool + Cipher sops.Cipher + ConfigPath string + InputPath string + KeyServices []keyservice.KeyServiceClient + InputStore sops.Store + OmitExtensions bool + Recursive bool + RootPath string } // Run publish operation @@ -42,14 +45,6 @@ func Run(opts Opts) error { if err != nil { return err } - info, err := os.Stat(path) - if err != nil { - return err - } - if info.IsDir() { - return fmt.Errorf("can't operate on a directory") - } - _, fileName := filepath.Split(path) conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string)) if err != nil { @@ -59,6 +54,19 @@ func Run(opts Opts) error { return errors.New("no destination configured for this file") } + var destinationPath string + if opts.Recursive { + destinationPath, err = filepath.Rel(opts.RootPath, opts.InputPath) + if err != nil { + return err + } + } else { + _, destinationPath = filepath.Split(path) + } + if opts.OmitExtensions || conf.OmitExtensions { + destinationPath = strings.TrimSuffix(destinationPath, filepath.Ext(path)) + } + // Check that this is a sops-encrypted file tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath) if err != nil { @@ -146,22 +154,28 @@ func Run(opts Opts) error { if opts.Interactive { var response string for response != "y" && response != "n" { - fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(fileName)) + fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(destinationPath)) _, err := fmt.Scanln(&response) if err != nil { return err } } if response == "n" { - return errors.New("Publish canceled") + msg := fmt.Sprintf("Publication of %s canceled", path) + if opts.Recursive { + fmt.Println(msg) + return nil + } else { + return errors.New(msg) + } } } switch dest := conf.Destination.(type) { case *publish.S3Destination, *publish.GCSDestination: - err = dest.Upload(fileContents, fileName) + err = dest.Upload(fileContents, destinationPath) case *publish.VaultDestination: - err = dest.UploadUnencrypted(data, fileName) + err = dest.UploadUnencrypted(data, destinationPath) } if err != nil { diff --git a/config/config.go b/config/config.go index 8e42c3a6b..62f48913d 100644 --- a/config/config.go +++ b/config/config.go @@ -100,6 +100,7 @@ type destinationRule struct { VaultKVMountName string `yaml:"vault_kv_mount_name"` VaultKVVersion int `yaml:"vault_kv_version"` RecreationRule creationRule `yaml:"recreation_rule,omitempty"` + OmitExtensions bool `yaml:"omit_extensions"` } type creationRule struct { @@ -133,6 +134,7 @@ type Config struct { EncryptedSuffix string EncryptedRegex string Destination publish.Destination + OmitExtensions bool } func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[string]*string) ([]sops.KeyGroup, error) { @@ -266,6 +268,7 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio return nil, err } config.Destination = dest + config.OmitExtensions = dRule.OmitExtensions return config, nil } diff --git a/publish/vault.go b/publish/vault.go index 01e0ddc0e..bfc5b3ba0 100644 --- a/publish/vault.go +++ b/publish/vault.go @@ -4,9 +4,19 @@ import ( "fmt" "strings" + "github.com/google/go-cmp/cmp" vault "github.com/hashicorp/vault/api" + "go.mozilla.org/sops/v3/logging" + + "github.com/sirupsen/logrus" ) +var log *logrus.Logger + +func init() { + log = logging.NewLogger("PUBLISH") +} + type VaultDestination struct { vaultAddress string vaultPath string @@ -65,6 +75,16 @@ func (vaultd *VaultDestination) UploadUnencrypted(data map[string]interface{}, f } } + secretsPath := vaultd.secretsPath(fileName) + existingSecret, err := client.Logical().Read(secretsPath) + if err != nil { + log.Warnf("Cannot check if destination secret already exists in %s. New version will be created even if the data has not been changed.", secretsPath) + } + if existingSecret != nil && cmp.Equal(data, existingSecret.Data["data"]) { + log.Infof("Secret in %s is already up-to-date.\n", secretsPath) + return nil + } + secretsData := make(map[string]interface{}) if vaultd.kvVersion == 1 { @@ -73,7 +93,7 @@ func (vaultd *VaultDestination) UploadUnencrypted(data map[string]interface{}, f secretsData["data"] = data } - _, err = client.Logical().Write(vaultd.secretsPath(fileName), secretsData) + _, err = client.Logical().Write(secretsPath, secretsData) if err != nil { return err }