Publishing improvements: directory walking; prevent Vault unneeded version increment

README.rst

@@ -968,6 +968,7 @@ This command requires a ``.sops.yaml`` configuration file. Below is an example:
         vault_kv_mount_name: "secret/" # default
         vault_kv_version: 2 # default
         path_regex: vault/*
+        omit_extensions: true
 
 The above configuration will place all files under ``s3/*`` into the S3 bucket ``sops-secrets``,
 all files under ``gcs/*`` into the GCS bucket ``sops-secrets``, and the contents of all files under
@@ -977,6 +978,11 @@ published to S3 and GCS, it will decrypt them and re-encrypt them using the
 
 You would deploy a file to S3 with a command like: ``sops publish s3/app.yaml``
 
+To publish all files in selected directory recursively, you need to specify ``--recurse`` flag.
+
+If you don't want file extension to appear in destination secret path, use ``--omit-extensions``
+flag or same ``.sops.yaml`` option.
+
 Publishing to Vault
 *******************
 
@@ -991,6 +997,9 @@ configuring the client.
 ``vault_kv_mount_name`` is used if your Vault KV is mounted somewhere other than ``secret/``.
 ``vault_kv_version`` supports ``1`` and ``2``, with ``2`` being the default.
 
+If destination secret path already exists in Vault and contains same data as source file, it will
+be skipped.
+
 Below is an example of publishing to Vault (using token auth with a local dev instance of Vault).
 
 .. code:: bash

cmd/sops/main.go

@@ -211,13 +211,21 @@ func main() {
 		},
 		{
 			Name:      "publish",
-			Usage:     "Publish sops file to a configured destination",
+			Usage:     "Publish sops file or directory to a configured destination",
 			ArgsUsage: `file`,
 			Flags: append([]cli.Flag{
 				cli.BoolFlag{
 					Name:  "yes, y",
 					Usage: `pre-approve all changes and run non-interactively`,
 				},
+				cli.BoolFlag{
+					Name:  "omit-extensions",
+					Usage: "Omit file extensions in destination path when publishing sops file to configured destinations",
+				},
+				cli.BoolFlag{
+					Name:  "recurse",
+					Usage: "If source path is directory, publish all its content recursively",
+				},
 				cli.BoolFlag{
 					Name:  "verbose",
 					Usage: "Enable verbose logging output",
@@ -235,14 +243,14 @@ func main() {
 					return common.NewExitError("Error: no file specified", codes.NoFileSpecified)
 				}
 				fileName := c.Args()[0]
-				inputStore := inputStore(c, fileName)
 				err = publishcmd.Run(publishcmd.Opts{
-					ConfigPath:  configPath,
-					InputPath:   fileName,
-					InputStore:  inputStore,
-					Cipher:      aes.NewCipher(),
-					KeyServices: keyservices(c),
-					Interactive: !c.Bool("yes"),
+					ConfigPath:     configPath,
+					InputPath:      fileName,
+					Cipher:         aes.NewCipher(),
+					KeyServices:    keyservices(c),
+					Interactive:    !c.Bool("yes"),
+					OmitExtensions: c.Bool("omit-extensions"),
+					Recurse:        c.Bool("recurse"),
 				})
 				if cliErr, ok := err.(*cli.ExitError); ok && cliErr != nil {
 					return cliErr

cmd/sops/subcommand/publish/publish.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"go.mozilla.org/sops/v3"
 	"go.mozilla.org/sops/v3/cmd/sops/codes"
@@ -27,12 +28,14 @@ func init() {
 
 // Opts represents publish options and config
 type Opts struct {
-	Interactive bool
-	Cipher      sops.Cipher
-	ConfigPath  string
-	InputPath   string
-	KeyServices []keyservice.KeyServiceClient
-	InputStore  sops.Store
+	Interactive    bool
+	Cipher         sops.Cipher
+	ConfigPath     string
+	InputPath      string
+	KeyServices    []keyservice.KeyServiceClient
+	InputStore     sops.Store
+	OmitExtensions bool
+	Recurse        bool
 }
 
 // Run publish operation
@@ -46,10 +49,27 @@ func Run(opts Opts) error {
 	if err != nil {
 		return err
 	}
-	if info.IsDir() {
+	if info.IsDir() && !opts.Recurse {
 		return fmt.Errorf("can't operate on a directory")
+	} else if info.IsDir() && opts.Recurse {
+		err = filepath.Walk(opts.InputPath, func(subPath string, info os.FileInfo, err error) error {
+			subAbsPath, _ := filepath.Abs(subPath)
+			if !info.IsDir() && subAbsPath != path {
+				subOpts := opts
+				subOpts.InputPath = subPath
+				return Run(subOpts)
+			} else {
+				return nil
+			}
+		})
+		if err != nil {
+			return err
+		}
+		return nil
 	}
-	_, fileName := filepath.Split(path)
+	fileSuffix := filepath.Ext(path)
+	opts.InputStore = common.DefaultStoreForPathOrFormat(path, fileSuffix)
+	destinationPath := opts.InputPath
 
 	conf, err := config.LoadDestinationRuleForFile(opts.ConfigPath, opts.InputPath, make(map[string]*string))
 	if err != nil {
@@ -58,6 +78,9 @@ func Run(opts Opts) error {
 	if conf.Destination == nil {
 		return errors.New("no destination configured for this file")
 	}
+	if opts.OmitExtensions || conf.OmitExtensions {
+		destinationPath = strings.TrimSuffix(destinationPath, fileSuffix)
+	}
 
 	// Check that this is a sops-encrypted file
 	tree, err := common.LoadEncryptedFile(opts.InputStore, opts.InputPath)
@@ -146,22 +169,28 @@ func Run(opts Opts) error {
 	if opts.Interactive {
 		var response string
 		for response != "y" && response != "n" {
-			fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(fileName))
+			fmt.Printf("uploading %s to %s ? (y/n): ", path, conf.Destination.Path(destinationPath))
 			_, err := fmt.Scanln(&response)
 			if err != nil {
 				return err
 			}
 		}
 		if response == "n" {
-			return errors.New("Publish canceled")
+			msg := fmt.Sprintf("Publication of %s canceled", path)
+			if opts.Recurse {
+				fmt.Println(msg)
+				return nil
+			} else {
+				return errors.New(msg)
+			}
 		}
 	}
 
 	switch dest := conf.Destination.(type) {
 	case *publish.S3Destination, *publish.GCSDestination:
-		err = dest.Upload(fileContents, fileName)
+		err = dest.Upload(fileContents, destinationPath)
 	case *publish.VaultDestination:
-		err = dest.UploadUnencrypted(data, fileName)
+		err = dest.UploadUnencrypted(data, destinationPath)
 	}
 
 	if err != nil {

config/config.go

@@ -100,6 +100,7 @@ type destinationRule struct {
 	VaultKVMountName string       `yaml:"vault_kv_mount_name"`
 	VaultKVVersion   int          `yaml:"vault_kv_version"`
 	RecreationRule   creationRule `yaml:"recreation_rule,omitempty"`
+	OmitExtensions   bool         `yaml:"omit_extensions"`
 }
 
 type creationRule struct {
@@ -133,6 +134,7 @@ type Config struct {
 	EncryptedSuffix   string
 	EncryptedRegex    string
 	Destination       publish.Destination
+	OmitExtensions    bool
 }
 
 func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[string]*string) ([]sops.KeyGroup, error) {
@@ -266,6 +268,7 @@ func parseDestinationRuleForFile(conf *configFile, filePath string, kmsEncryptio
 		return nil, err
 	}
 	config.Destination = dest
+	config.OmitExtensions = dRule.OmitExtensions
 
 	return config, nil
 }

publish/vault.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"strings"
 
+	"github.com/google/go-cmp/cmp"
 	vault "github.com/hashicorp/vault/api"
 )
 
@@ -65,6 +66,17 @@ func (vaultd *VaultDestination) UploadUnencrypted(data map[string]interface{}, f
 		}
 	}
 
+	existingSecret, err := client.Logical().Read(vaultd.secretsPath(fileName))
+	if err != nil {
+		return err
+	}
+	if existingSecret != nil {
+		if cmp.Equal(data, existingSecret.Data["data"]) {
+			fmt.Printf("Secret in %s is already up-to-date.\n", vaultd.secretsPath(fileName))
+			return nil
+		}
+	}
+
 	secretsData := make(map[string]interface{})
 
 	if vaultd.kvVersion == 1 {