diff --git a/.github/workflows/cli.yml b/.github/workflows/cli.yml new file mode 100644 index 000000000..30ffde52f --- /dev/null +++ b/.github/workflows/cli.yml @@ -0,0 +1,77 @@ +name: CLI + +on: + push: + branches: [develop] + pull_request: + branches: [develop] + +jobs: + build: + name: Build and test ${{ matrix.os }} + runs-on: ubuntu-latest + strategy: + matrix: + os: [linux, darwin, windows] + env: + VAULT_VERSION: "1.1.3" + VAULT_TOKEN: "root" + VAULT_ADDR: "http://127.0.0.1:8200" + steps: + - name: Install dependencies + run: sudo apt-get update && sudo apt-get install git -y + - name: Set up Go 1.13 + uses: actions/setup-go@v2 + with: + go-version: 1.13 + id: go + - name: Check out code into the Go module directory + uses: actions/checkout@v2 + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Build + run: GOOS=${{ matrix.os }} go build -o sops-${{ matrix.os }}-${{ github.sha }} -v ./cmd/sops + - name: Import test GPG keys + run: for i in 1 2 3 4 5; do gpg --import pgp/sops_functional_tests_key.asc && break || sleep 15; done + - name: Test + run: make test + - name: Upload artifact + uses: actions/upload-artifact@v2 + with: + name: sops-${{ matrix.os }}-${{ github.sha }} + path: sops-${{ matrix.os }}-${{ github.sha }} + test: + name: Functional tests + runs-on: ubuntu-latest + needs: [build] + env: + VAULT_VERSION: "1.1.3" + VAULT_TOKEN: "root" + VAULT_ADDR: "http://127.0.0.1:8200" + steps: + - name: Install rustup + run: curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | bash -s -- -y --default-toolchain 1.47.0 + - name: Check out code + uses: actions/checkout@v2 + - uses: actions/download-artifact@v2 + with: + name: sops-linux-${{ github.sha }} + - name: Move SOPS binary + run: mv sops-linux-${{ github.sha }} ./functional-tests/sops + - name: Make SOPS binary executable + run: chmod +x ./functional-tests/sops + - name: Download Vault + run: curl -O "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip" && sudo unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ + - name: Start Vault server + run: vault server -dev -dev-root-token-id="$VAULT_TOKEN" & + - name: Enable Vault KV + run: vault secrets enable -version=1 kv + - name: Import test GPG keys + run: for i in 1 2 3 4 5; do gpg --import pgp/sops_functional_tests_key.asc && break || sleep 15; done + - name: Run tests + run: cargo test + working-directory: ./functional-tests diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index dd5307efe..000000000 --- a/.travis.yml +++ /dev/null @@ -1,55 +0,0 @@ -language: go -go: 1.13 -go_import_path: go.mozilla.org/sops/ - -env: - - VAULT_VERSION=1.1.3 VAULT_TOKEN=root VAULT_ADDR='http://127.0.0.1:8200' - -addons: - apt: - packages: - - rpm - - ruby - - python3 - - unzip - -before_install: - - gem install fpm || sudo gem install fpm - - curl https://sh.rustup.rs -sSf | sh -s -- -y - - source ~/.cargo/env - - curl -O "https://releases.hashicorp.com/vault/${VAULT_VERSION}/vault_${VAULT_VERSION}_linux_amd64.zip" && sudo unzip vault_${VAULT_VERSION}_linux_amd64.zip -d /usr/local/bin/ - -before_script: - - vault server -dev -dev-root-token-id="$VAULT_TOKEN" & - - sleep 5 - - vault secrets enable -version=1 kv - -script: - - 'if [ "$TRAVIS_PULL_REQUEST" != "false" ]; then make; fi' - - 'if [ "$TRAVIS_PULL_REQUEST" = "false" ]; then make origin-build; fi' - - bash <(curl -s https://codecov.io/bash) - -before_deploy: - - mkdir dist - - make deb-pkg rpm-pkg - - mv *.deb *.rpm dist/ - - GOOS=darwin CGO_ENABLED=0 GO111MODULE=on go build -mod vendor -o dist/sops-${TRAVIS_TAG}.darwin go.mozilla.org/sops/v3/cmd/sops - - GOOS=windows CGO_ENABLED=0 GO111MODULE=on go build -mod vendor -o dist/sops-${TRAVIS_TAG}.exe go.mozilla.org/sops/v3/cmd/sops - - GOOS=linux CGO_ENABLED=0 GO111MODULE=on go build -mod vendor -o dist/sops-${TRAVIS_TAG}.linux go.mozilla.org/sops/v3/cmd/sops - - | - if [ ! -z "$TRAVIS_TAG" ]; then - version="v$(grep '^const Version' version/version.go |cut -d '"' -f 2)" - if [ "$version" != "$TRAVIS_TAG" ]; then - echo "Git tag $TRAVIS_TAG does not match version $version, update the source!" - exit 1 - fi - fi - -deploy: - provider: releases - api_key: "${GITHUB_OAUTH_TOKEN}" - file_glob: true - file: dist/* - skip_cleanup: true - on: - tags: true diff --git a/CHANGELOG.rst b/CHANGELOG.rst index 48e777227..5a3049738 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -1,6 +1,23 @@ Changelog ========= +3.7.0 +----- +Features: + + * Add support for age (#688) + * Add filename to exec-file (#761) + +Changes: + + * On failed decryption with GPG, return the error returned by GPG to the sops user (#762) + * Use yaml.v3 instead of modified yaml.v2 for handling YAML files (#791) + * Update aws-sdk-go to version v1.37.18 (#823) + +Project Changes: + + * Switch from TravisCI to Github Actions (#792) + 3.6.1 ----- Features: diff --git a/README.rst b/README.rst index 603356c21..b05fcf107 100644 --- a/README.rst +++ b/README.rst @@ -2,7 +2,7 @@ SOPS: Secrets OPerationS ======================== **sops** is an editor of encrypted files that supports YAML, JSON, ENV, INI and BINARY -formats and encrypts with AWS KMS, GCP KMS, Azure Key Vault and PGP. +formats and encrypts with AWS KMS, GCP KMS, Azure Key Vault, age, and PGP. (`demo `_) .. image:: https://i.imgur.com/X0TM5NI.gif @@ -178,6 +178,33 @@ the example files and pgp key provided with the repository:: This last step will decrypt ``example.yaml`` using the test private key. +Encrypting using age +~~~~~~~~~~~~~~~~~~~~ + +`age `_ is a simple, modern, and secure tool for +encrypting files. It's recommended to use age over PGP, if possible. + +You can encrypt a file for one or more age recipients (comma separated) using +the ``--age`` option or the **SOPS_AGE_RECIPIENTS** environment variable: + +.. code:: bash + + $ sops --age age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw test.yaml > test.enc.yaml + +When decrypting a file with the corresponding identity, sops will look for a +text file name ``keys.txt`` located in a ``sops`` subdirectory of your user +configuration directory. On Linux, this would be ``$XDG_CONFIG_HOME/sops/keys.txt``. +On macOS, this would be ``$HOME/Library/Application Support/sops/keys.txt``. On +Windows, this would be ``%AppData%\sops\keys.txt``. You can specify the location +of this file manually by setting the environment variable **SOPS_AGE_KEY_FILE**. + +The contents of this key file should be a list of age X25519 identities, one +per line. Lines beginning with ``#`` are considered comments and ignored. Each +identity will be tried in sequence until one is able to decrypt the data. + +Encrypting with SSH keys via age is not yet supported by sops. + + Encrypting using GCP KMS ~~~~~~~~~~~~~~~~~~~~~~~~ GCP KMS uses `Application Default Credentials @@ -372,7 +399,7 @@ The sops team recommends the ``updatekeys`` approach. ``updatekeys`` command ********************** -The ``updatekeys`` command uses the `.sops.yaml <#29using-sopsyaml-conf-to-select-kmspgp-for-new-files>`_ +The ``updatekeys`` command uses the `.sops.yaml <#using-sops-yaml-conf-to-select-kms-pgp-for-new-files>`_ configuration file to update (add or remove) the corresponding secrets in the encrypted file. Note that the example below uses the `Block Scalar yaml construct `_ to build a space @@ -675,7 +702,7 @@ Specify a different GPG key server By default, ``sops`` uses the key server ``keys.openpgp.org`` to retrieve the GPG keys that are not present in the local keyring. -This is no longer configurable. You can learn more about why from this write-up: [SKS Keyserver Network Under Attack](https://gist.github.com/rjhansen/67ab921ffb4084c865b3618d6955275f). +This is no longer configurable. You can learn more about why from this write-up: `SKS Keyserver Network Under Attack `_. Example: place the following in your ``~/.bashrc`` @@ -986,6 +1013,9 @@ encrypted file is only readable by root, but the target program does not need root privileges to function. This flag should be used where possible for added security. +To overwrite the default file name (``tmp-file``) in ``exec-file`` use the +``--filename `` parameter. + .. code:: bash # the encrypted file can't be read by the current user @@ -1402,7 +1432,7 @@ By default, ``sops`` encrypts all the values of a YAML or JSON file and leaves t keys in cleartext. In some instances, you may want to exclude some values from being encrypted. This can be accomplished by adding the suffix **_unencrypted** to any key of a file. When set, all values underneath the key that set the -**_unencrypted** prefix will be left in cleartext. +**_unencrypted** suffix will be left in cleartext. Note that, while in cleartext, unencrypted content is still added to the checksum of the file, and thus cannot be modified outside of sops without diff --git a/age/keys.txt b/age/keys.txt new file mode 100644 index 000000000..c56eb4fab --- /dev/null +++ b/age/keys.txt @@ -0,0 +1,3 @@ +# created: 2020-07-18T03:16:47-07:00 +# public key: age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw +AGE-SECRET-KEY-1NJT5YCS2LWU4V4QAJQ6R4JNU7LXPDX602DZ9NUFANVU5GDTGUWCQ5T59M6 diff --git a/age/keysource.go b/age/keysource.go new file mode 100644 index 000000000..c0b9c6561 --- /dev/null +++ b/age/keysource.go @@ -0,0 +1,203 @@ +package age + +import ( + "bytes" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + "filippo.io/age" + "filippo.io/age/armor" + "github.com/sirupsen/logrus" + "go.mozilla.org/sops/v3/logging" +) + +var log *logrus.Logger + +func init() { + log = logging.NewLogger("AGE") +} + +const privateKeySizeLimit = 1 << 24 // 16 MiB + +// MasterKey is an age key used to encrypt and decrypt sops' data key. +type MasterKey struct { + Identity string // a Bech32-encoded private key + Recipient string // a Bech32-encoded public key + EncryptedKey string // a sops data key encrypted with age + + parsedIdentity *age.X25519Identity // a parsed age private key + parsedRecipient *age.X25519Recipient // a parsed age public key +} + +// Encrypt takes a sops data key, encrypts it with age and stores the result in the EncryptedKey field. +func (key *MasterKey) Encrypt(datakey []byte) error { + buffer := &bytes.Buffer{} + + if key.parsedRecipient == nil { + parsedRecipient, err := parseRecipient(key.Recipient) + + if err != nil { + log.WithField("recipient", key.parsedRecipient).Error("Encryption failed") + return err + } + + key.parsedRecipient = parsedRecipient + } + + aw := armor.NewWriter(buffer) + w, err := age.Encrypt(aw, key.parsedRecipient) + if err != nil { + return fmt.Errorf("failed to open file for encrypting sops data key with age: %w", err) + } + + if _, err := w.Write(datakey); err != nil { + log.WithField("recipient", key.parsedRecipient).Error("Encryption failed") + return fmt.Errorf("failed to encrypt sops data key with age: %w", err) + } + + if err := w.Close(); err != nil { + log.WithField("recipient", key.parsedRecipient).Error("Encryption failed") + return fmt.Errorf("failed to close file for encrypting sops data key with age: %w", err) + } + + if err := aw.Close(); err != nil { + log.WithField("recipient", key.parsedRecipient).Error("Encryption failed") + return fmt.Errorf("failed to close armored writer: %w", err) + } + + key.EncryptedKey = buffer.String() + + log.WithField("recipient", key.parsedRecipient).Info("Encryption succeeded") + + return nil +} + +// EncryptIfNeeded encrypts the provided sops' data key and encrypts it if it hasn't been encrypted yet. +func (key *MasterKey) EncryptIfNeeded(datakey []byte) error { + if key.EncryptedKey == "" { + return key.Encrypt(datakey) + } + + return nil +} + +// EncryptedDataKey returns the encrypted data key this master key holds. +func (key *MasterKey) EncryptedDataKey() []byte { + return []byte(key.EncryptedKey) +} + +// SetEncryptedDataKey sets the encrypted data key for this master key. +func (key *MasterKey) SetEncryptedDataKey(enc []byte) { + key.EncryptedKey = string(enc) +} + +// Decrypt decrypts the EncryptedKey field with the age identity and returns the result. +func (key *MasterKey) Decrypt() ([]byte, error) { + ageKeyFilePath, ok := os.LookupEnv("SOPS_AGE_KEY_FILE") + + if !ok { + userConfigDir, err := os.UserConfigDir() + + if err != nil { + return nil, fmt.Errorf("user config directory could not be determined: %w", err) + } + + ageKeyFilePath = filepath.Join(userConfigDir, "sops", "age", "keys.txt") + } + + ageKeyFile, err := os.Open(ageKeyFilePath) + + if err != nil { + return nil, fmt.Errorf("failed to open file: %w", err) + } + + defer ageKeyFile.Close() + + identities, err := age.ParseIdentities(ageKeyFile) + + if err != nil { + return nil, err + } + + src := bytes.NewReader([]byte(key.EncryptedKey)) + ar := armor.NewReader(src) + r, err := age.Decrypt(ar, identities...) + + if err != nil { + return nil, fmt.Errorf("no age identity found in %q that could decrypt the data", ageKeyFilePath) + } + + var b bytes.Buffer + if _, err := io.Copy(&b, r); err != nil { + return nil, fmt.Errorf("failed to copy decrypted data into bytes.Buffer: %w", err) + } + + return b.Bytes(), nil +} + +// NeedsRotation returns whether the data key needs to be rotated or not. +func (key *MasterKey) NeedsRotation() bool { + return false +} + +// ToString converts the key to a string representation. +func (key *MasterKey) ToString() string { + return key.Recipient +} + +// ToMap converts the MasterKey to a map for serialization purposes. +func (key *MasterKey) ToMap() map[string]interface{} { + return map[string]interface{}{"recipient": key.Recipient, "enc": key.EncryptedKey} +} + +// MasterKeysFromRecipients takes a comma-separated list of Bech32-encoded public keys and returns a +// slice of new MasterKeys. +func MasterKeysFromRecipients(commaSeparatedRecipients string) ([]*MasterKey, error) { + if commaSeparatedRecipients == "" { + // otherwise Split returns [""] and MasterKeyFromRecipient is unhappy + return make([]*MasterKey, 0), nil + } + recipients := strings.Split(commaSeparatedRecipients, ",") + + var keys []*MasterKey + + for _, recipient := range recipients { + key, err := MasterKeyFromRecipient(recipient) + + if err != nil { + return nil, err + } + + keys = append(keys, key) + } + + return keys, nil +} + +// MasterKeyFromRecipient takes a Bech32-encoded public key and returns a new MasterKey. +func MasterKeyFromRecipient(recipient string) (*MasterKey, error) { + parsedRecipient, err := parseRecipient(recipient) + + if err != nil { + return nil, err + } + + return &MasterKey{ + Recipient: recipient, + parsedRecipient: parsedRecipient, + }, nil +} + +// parseRecipient attempts to parse a string containing an encoded age public key +func parseRecipient(recipient string) (*age.X25519Recipient, error) { + parsedRecipient, err := age.ParseX25519Recipient(recipient) + + if err != nil { + return nil, fmt.Errorf("failed to parse input as Bech32-encoded age public key: %w", err) + } + + return parsedRecipient, nil +} diff --git a/age/keysource_test.go b/age/keysource_test.go new file mode 100644 index 000000000..2a3bebdfd --- /dev/null +++ b/age/keysource_test.go @@ -0,0 +1,69 @@ +package age + +import ( + "os" + "path" + "runtime" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestMasterKeysFromRecipientsEmpty(t *testing.T) { + assert := assert.New(t) + + commaSeparatedRecipients := "" + recipients, err := MasterKeysFromRecipients(commaSeparatedRecipients) + + assert.NoError(err) + + assert.Equal(recipients, make([]*MasterKey, 0)) +} + +func TestAge(t *testing.T) { + assert := assert.New(t) + + key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw") + + assert.NoError(err) + assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString()) + + dataKey := []byte("abcdefghijklmnopqrstuvwxyz123456") + + err = key.Encrypt(dataKey) + assert.NoError(err) + + _, filename, _, _ := runtime.Caller(0) + err = os.Setenv("SOPS_AGE_KEY_FILE", path.Join(path.Dir(filename), "keys.txt")) + assert.NoError(err) + + decryptedKey, err := key.Decrypt() + assert.NoError(err) + assert.Equal(dataKey, decryptedKey) +} + +func TestAgeDotEnv(t *testing.T) { + assert := assert.New(t) + + key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw") + + assert.NoError(err) + assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString()) + + dotenv := `IMAGE_PREFIX=repo/service- +APPLICATION_KEY=K6pfAWuUVND9Fz5SC7jmA6pfAWuUVND9Fz5SC7jmA +KEY_ID=003683d721f2ae683d721f2a1 +DOMAIN=files.127.0.0.1.nip.io` + dataKey := []byte(dotenv) + + err = key.Encrypt(dataKey) + assert.NoError(err) + + _, filename, _, _ := runtime.Caller(0) + err = os.Setenv("SOPS_AGE_KEY_FILE", path.Join(path.Dir(filename), "keys.txt")) + assert.NoError(err) + + decryptedKey, err := key.Decrypt() + assert.NoError(err) + assert.Equal(dataKey, decryptedKey) +} diff --git a/audit/audit.go b/audit/audit.go index 1a29f5ea2..b52215077 100644 --- a/audit/audit.go +++ b/audit/audit.go @@ -12,7 +12,7 @@ import ( // empty import as per https://godoc.org/github.com/lib/pq _ "github.com/lib/pq" - "github.com/mozilla-services/yaml" + "gopkg.in/yaml.v3" "github.com/sirupsen/logrus" "go.mozilla.org/sops/v3/logging" ) diff --git a/azkv/keysource.go b/azkv/keysource.go index 85a549b7f..127ccccf8 100644 --- a/azkv/keysource.go +++ b/azkv/keysource.go @@ -210,7 +210,7 @@ func (key *MasterKey) Encrypt(dataKey []byte) error { "key": key.Name, "version": key.Version, }).Error("Encryption failed") - return fmt.Errorf("Failed to encrypt data: %v", err) + return fmt.Errorf("Failed to encrypt data: %w", err) } key.EncryptedKey = *res.Result @@ -244,7 +244,7 @@ func (key *MasterKey) Decrypt() ([]byte, error) { "key": key.Name, "version": key.Version, }).Error("Decryption failed") - return nil, fmt.Errorf("Error decrypting key: %v", err) + return nil, fmt.Errorf("Error decrypting key: %w", err) } plaintext, err := base64.RawURLEncoding.DecodeString(*res.Result) diff --git a/cmd/sops/main.go b/cmd/sops/main.go index e99733d8e..148daa729 100644 --- a/cmd/sops/main.go +++ b/cmd/sops/main.go @@ -16,6 +16,7 @@ import ( "github.com/sirupsen/logrus" "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/aes" + "go.mozilla.org/sops/v3/age" _ "go.mozilla.org/sops/v3/audit" "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/cmd/sops/codes" @@ -61,7 +62,7 @@ func main() { }, } app.Name = "sops" - app.Usage = "sops - encrypted file editor with AWS KMS, GCP KMS, Azure Key Vault and GPG support" + app.Usage = "sops - encrypted file editor with AWS KMS, GCP KMS, Azure Key Vault, age, and GPG support" app.ArgsUsage = "sops [options] file" app.Version = version.Version app.Authors = []cli.Author{ @@ -96,6 +97,9 @@ func main() { https://docs.microsoft.com/en-us/go/azure/azure-sdk-go-authorization#use-environment-based-authentication. The user/sp needs the key/encrypt and key/decrypt permissions) + To encrypt or decrypt using age, specify the recipient in the -a flag, or + in the SOPS_AGE_RECIPIENTS environment variable. + To encrypt or decrypt using PGP, specify the PGP fingerprint in the -p flag or in the SOPS_PGP_FP environment variable. @@ -191,6 +195,10 @@ func main() { Name: "output-type", Usage: "currently json, yaml, dotenv and binary are supported. If not set, sops will use the input file's extension to determine the output format", }, + cli.StringFlag{ + Name: "filename", + Usage: "filename for the temporarily file (default: tmp-file)", + }, }, keyserviceFlags...), Action: func(c *cli.Context) error { if len(c.Args()) != 2 { @@ -218,12 +226,18 @@ func main() { return toExitError(err) } + filename := c.String("filename") + if filename == "" { + filename = "tmp-file" + } + if err := exec.ExecWithFile(exec.ExecOpts{ Command: command, Plaintext: output, Background: c.Bool("background"), Fifo: !c.Bool("no-fifo"), User: c.String("user"), + Filename: filename, }); err != nil { return toExitError(err) } @@ -377,6 +391,10 @@ func main() { Name: "hc-vault-transit", Usage: "the full vault path to the key used to encrypt/decrypt. Make you choose and configure a key with encrption/decryption enabled (e.g. 'https://vault.example.org:8200/v1/transit/keys/dev'). Can be specified more than once", }, + cli.StringSliceFlag{ + Name: "age", + Usage: "the age recipient the new group should contain. Can be specified more than once", + }, cli.BoolFlag{ Name: "in-place, i", Usage: "write output back to the same file instead of stdout", @@ -396,6 +414,7 @@ func main() { gcpKmses := c.StringSlice("gcp-kms") vaultURIs := c.StringSlice("hc-vault-transit") azkvs := c.StringSlice("azure-kv") + ageRecipients := c.StringSlice("age") var group sops.KeyGroup for _, fp := range pgpFps { group = append(group, pgp.NewMasterKeyFromFingerprint(fp)) @@ -422,6 +441,14 @@ func main() { } group = append(group, k) } + for _, recipient := range ageRecipients { + k, err := age.MasterKeyFromRecipient(recipient) + if err != nil { + log.WithError(err).Error("Failed to add key") + continue + } + group = append(group, k) + } return groups.Add(groups.AddOpts{ InputPath: c.String("file"), InPlace: c.Bool("in-place"), @@ -552,6 +579,11 @@ func main() { Usage: "comma separated list of PGP fingerprints", EnvVar: "SOPS_PGP_FP", }, + cli.StringFlag{ + Name: "age, a", + Usage: "comma separated list of age recipients", + EnvVar: "SOPS_AGE_RECIPIENTS", + }, cli.BoolFlag{ Name: "in-place, i", Usage: "write output back to the same file instead of stdout", @@ -604,6 +636,14 @@ func main() { Name: "rm-hc-vault-transit", Usage: "remove the provided comma-separated list of Vault's URI key from the list of master keys on the given file ( eg. https://vault.example.org:8200/v1/transit/keys/dev)", }, + cli.StringFlag{ + Name: "add-age", + Usage: "add the provided comma-separated list of age recipients fingerprints to the list of master keys on the given file", + }, + cli.StringFlag{ + Name: "rm-age", + Usage: "remove the provided comma-separated list of age recipients from the list of master keys on the given file", + }, cli.StringFlag{ Name: "add-pgp", Usage: "add the provided comma-separated list of PGP fingerprints to the list of master keys on the given file", @@ -673,8 +713,8 @@ func main() { return toExitError(err) } if _, err := os.Stat(fileName); os.IsNotExist(err) { - if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-hc-vault-transit") != "" || c.String("add-azure-kv") != "" || - c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-hc-vault-transit") != "" || c.String("rm-azure-kv") != "" { + if c.String("add-kms") != "" || c.String("add-pgp") != "" || c.String("add-gcp-kms") != "" || c.String("add-hc-vault-transit") != "" || c.String("add-azure-kv") != "" || c.String("add-age") != "" || + c.String("rm-kms") != "" || c.String("rm-pgp") != "" || c.String("rm-gcp-kms") != "" || c.String("rm-hc-vault-transit") != "" || c.String("rm-azure-kv") != "" || c.String("rm-age") != "" { return common.NewExitError("Error: cannot add or remove keys on non-existent files, use `--kms` and `--pgp` instead.", codes.CannotChangeKeysFromNonExistentFile) } if c.Bool("encrypt") || c.Bool("decrypt") || c.Bool("rotate") { @@ -802,6 +842,13 @@ func main() { for _, k := range hcVaultKeys { addMasterKeys = append(addMasterKeys, k) } + ageKeys, err := age.MasterKeysFromRecipients(c.String("add-age")) + if err != nil { + return err + } + for _, k := range ageKeys { + addMasterKeys = append(addMasterKeys, k) + } var rmMasterKeys []keys.MasterKey for _, k := range kms.MasterKeysFromArnString(c.String("rm-kms"), kmsEncryptionContext, c.String("aws-profile")) { @@ -827,6 +874,13 @@ func main() { for _, k := range hcVaultKeys { rmMasterKeys = append(rmMasterKeys, k) } + ageKeys, err = age.MasterKeysFromRecipients(c.String("rm-age")) + if err != nil { + return err + } + for _, k := range ageKeys { + rmMasterKeys = append(rmMasterKeys, k) + } output, err = rotate(rotateOpts{ OutputStore: outputStore, @@ -1023,6 +1077,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { var cloudKmsKeys []keys.MasterKey var azkvKeys []keys.MasterKey var hcVaultMkKeys []keys.MasterKey + var ageMasterKeys []keys.MasterKey kmsEncryptionContext := kms.ParseKMSContext(c.String("encryption-context")) if c.String("encryption-context") != "" && kmsEncryptionContext == nil { return nil, common.NewExitError("Invalid KMS encryption context format", codes.ErrorInvalidKMSEncryptionContextFormat) @@ -1060,7 +1115,16 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { pgpKeys = append(pgpKeys, k) } } - if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" && c.String("hc-vault-transit") == "" { + if c.String("age") != "" { + ageKeys, err := age.MasterKeysFromRecipients(c.String("age")) + if err != nil { + return nil, err + } + for _, k := range ageKeys { + ageMasterKeys = append(ageMasterKeys, k) + } + } + if c.String("kms") == "" && c.String("pgp") == "" && c.String("gcp-kms") == "" && c.String("azure-kv") == "" && c.String("hc-vault-transit") == "" && c.String("age") == "" { conf, err := loadConfig(c, file, kmsEncryptionContext) // config file might just not be supplied, without any error if conf == nil { @@ -1078,6 +1142,7 @@ func keyGroups(c *cli.Context, file string) ([]sops.KeyGroup, error) { group = append(group, azkvKeys...) group = append(group, pgpKeys...) group = append(group, hcVaultMkKeys...) + group = append(group, ageMasterKeys...) log.Debugf("Master keys available: %+v", group) return []sops.KeyGroup{group}, nil } diff --git a/cmd/sops/subcommand/exec/exec.go b/cmd/sops/subcommand/exec/exec.go index 95f135d79..cd8d33be5 100644 --- a/cmd/sops/subcommand/exec/exec.go +++ b/cmd/sops/subcommand/exec/exec.go @@ -24,10 +24,11 @@ type ExecOpts struct { Background bool Fifo bool User string + Filename string } -func GetFile(dir string) *os.File { - handle, err := ioutil.TempFile(dir, "tmp-file") +func GetFile(dir, filename string) *os.File { + handle, err := ioutil.TempFile(dir, filename) if err != nil { log.Fatal(err) } @@ -54,10 +55,10 @@ func ExecWithFile(opts ExecOpts) error { if opts.Fifo { // fifo handling needs to be async, even opening to write // will block if there is no reader present - filename = GetPipe(dir) + filename = GetPipe(dir, opts.Filename) go WritePipe(filename, opts.Plaintext) } else { - handle := GetFile(dir) + handle := GetFile(dir, opts.Filename) handle.Write(opts.Plaintext) handle.Close() filename = handle.Name() diff --git a/cmd/sops/subcommand/exec/exec_unix.go b/cmd/sops/subcommand/exec/exec_unix.go index f2041b7fb..cc831e798 100644 --- a/cmd/sops/subcommand/exec/exec_unix.go +++ b/cmd/sops/subcommand/exec/exec_unix.go @@ -27,8 +27,8 @@ func WritePipe(pipe string, contents []byte) { handle.Close() } -func GetPipe(dir string) string { - tmpfn := filepath.Join(dir, "tmp-file") +func GetPipe(dir, filename string) string { + tmpfn := filepath.Join(dir, filename) err := syscall.Mkfifo(tmpfn, 0600) if err != nil { log.Fatal(err) diff --git a/cmd/sops/subcommand/exec/exec_windows.go b/cmd/sops/subcommand/exec/exec_windows.go index 0c3345384..7e0f21d74 100644 --- a/cmd/sops/subcommand/exec/exec_windows.go +++ b/cmd/sops/subcommand/exec/exec_windows.go @@ -12,7 +12,7 @@ func WritePipe(pipe string, contents []byte) { log.Fatal("fifos are not available on windows") } -func GetPipe(dir string) string { +func GetPipe(dir, filename string) string { log.Fatal("fifos are not available on windows") return "" } diff --git a/config/config.go b/config/config.go index 67a06cdda..e89336ddc 100644 --- a/config/config.go +++ b/config/config.go @@ -10,9 +10,9 @@ import ( "path" "regexp" - "github.com/mozilla-services/yaml" "github.com/sirupsen/logrus" "go.mozilla.org/sops/v3" + "go.mozilla.org/sops/v3/age" "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/gcpkms" "go.mozilla.org/sops/v3/hcvault" @@ -20,6 +20,7 @@ import ( "go.mozilla.org/sops/v3/logging" "go.mozilla.org/sops/v3/pgp" "go.mozilla.org/sops/v3/publish" + "gopkg.in/yaml.v3" ) var log *logrus.Logger @@ -71,6 +72,7 @@ type keyGroup struct { GCPKMS []gcpKmsKey `yaml:"gcp_kms"` AzureKV []azureKVKey `yaml:"azure_keyvault"` Vault []string `yaml:"hc_vault"` + Age []string `yaml:"age"` PGP []string } @@ -109,6 +111,7 @@ type creationRule struct { PathRegex string `yaml:"path_regex"` KMS string AwsProfile string `yaml:"aws_profile"` + Age string `yaml:"age"` PGP string GCPKMS string `yaml:"gcp_kms"` AzureKeyVault string `yaml:"azure_keyvault"` @@ -147,6 +150,13 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ if len(cRule.KeyGroups) > 0 { for _, group := range cRule.KeyGroups { var keyGroup sops.KeyGroup + for _, k := range group.Age { + key, err := age.MasterKeyFromRecipient(k) + if err != nil { + return nil, err + } + keyGroup = append(keyGroup, key) + } for _, k := range group.PGP { keyGroup = append(keyGroup, pgp.NewMasterKeyFromFingerprint(k)) } @@ -170,6 +180,16 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[ } } else { var keyGroup sops.KeyGroup + if cRule.Age != "" { + ageKeys, err := age.MasterKeysFromRecipients(cRule.Age) + if err != nil { + return nil, err + } else { + for _, ak := range ageKeys { + keyGroup = append(keyGroup, ak) + } + } + } for _, k := range pgp.MasterKeysFromFingerprintString(cRule.PGP) { keyGroup = append(keyGroup, k) } @@ -306,11 +326,13 @@ func parseCreationRuleForFile(conf *configFile, filePath string, kmsEncryptionCo rule = &r break } - if r.PathRegex != "" { - if match, _ := regexp.MatchString(r.PathRegex, filePath); match { - rule = &r - break - } + reg, err := regexp.Compile(r.PathRegex) + if err != nil { + return nil, fmt.Errorf("can not compile regexp: %w", err) + } + if reg.MatchString(filePath) { + rule = &r + break } } diff --git a/config/config_test.go b/config/config_test.go index 183eb7836..ac8aca6f3 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -208,6 +208,24 @@ destination_rules: path_regex: "vault-v1/*" `) +var sampleConfigWithInvalidComplicatedRegexp = []byte(` +creation_rules: + - path_regex: "[ ]\\K(? KmsKey + 1, // 1: Key.pgp_key:type_name -> PgpKey + 3, // 2: Key.gcp_kms_key:type_name -> GcpKmsKey + 5, // 3: Key.azure_keyvault_key:type_name -> AzureKeyVaultKey + 4, // 4: Key.vault_key:type_name -> VaultKey + 6, // 5: Key.age_key:type_name -> AgeKey + 11, // 6: KmsKey.context:type_name -> KmsKey.ContextEntry + 0, // 7: EncryptRequest.key:type_name -> Key + 0, // 8: DecryptRequest.key:type_name -> Key + 7, // 9: KeyService.Encrypt:input_type -> EncryptRequest + 9, // 10: KeyService.Decrypt:input_type -> DecryptRequest + 8, // 11: KeyService.Encrypt:output_type -> EncryptResponse + 10, // 12: KeyService.Decrypt:output_type -> DecryptResponse + 11, // [11:13] is the sub-list for method output_type + 9, // [9:11] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name +} + +func init() { file_keyservice_keyservice_proto_init() } +func file_keyservice_keyservice_proto_init() { + if File_keyservice_keyservice_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_keyservice_keyservice_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*Key); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*PgpKey); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[2].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*KmsKey); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[3].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GcpKmsKey); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[4].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*VaultKey); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[5].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AzureKeyVaultKey); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[6].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*AgeKey); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[7].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EncryptRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[8].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*EncryptResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DecryptRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_keyservice_keyservice_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*DecryptResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + file_keyservice_keyservice_proto_msgTypes[0].OneofWrappers = []interface{}{ + (*Key_KmsKey)(nil), + (*Key_PgpKey)(nil), + (*Key_GcpKmsKey)(nil), + (*Key_AzureKeyvaultKey)(nil), + (*Key_VaultKey)(nil), + (*Key_AgeKey)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_keyservice_keyservice_proto_rawDesc, + NumEnums: 0, + NumMessages: 12, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_keyservice_keyservice_proto_goTypes, + DependencyIndexes: file_keyservice_keyservice_proto_depIdxs, + MessageInfos: file_keyservice_keyservice_proto_msgTypes, + }.Build() + File_keyservice_keyservice_proto = out.File + file_keyservice_keyservice_proto_rawDesc = nil + file_keyservice_keyservice_proto_goTypes = nil + file_keyservice_keyservice_proto_depIdxs = nil } // Reference imports to suppress errors if they are not otherwise used. var _ context.Context -var _ grpc.ClientConn +var _ grpc.ClientConnInterface // This is a compile-time assertion to ensure that this generated file // is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 +const _ = grpc.SupportPackageIsVersion6 // KeyServiceClient is the client API for KeyService service. // @@ -646,10 +1019,10 @@ type KeyServiceClient interface { } type keyServiceClient struct { - cc *grpc.ClientConn + cc grpc.ClientConnInterface } -func NewKeyServiceClient(cc *grpc.ClientConn) KeyServiceClient { +func NewKeyServiceClient(cc grpc.ClientConnInterface) KeyServiceClient { return &keyServiceClient{cc} } @@ -681,10 +1054,10 @@ type KeyServiceServer interface { type UnimplementedKeyServiceServer struct { } -func (*UnimplementedKeyServiceServer) Encrypt(ctx context.Context, req *EncryptRequest) (*EncryptResponse, error) { +func (*UnimplementedKeyServiceServer) Encrypt(context.Context, *EncryptRequest) (*EncryptResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Encrypt not implemented") } -func (*UnimplementedKeyServiceServer) Decrypt(ctx context.Context, req *DecryptRequest) (*DecryptResponse, error) { +func (*UnimplementedKeyServiceServer) Decrypt(context.Context, *DecryptRequest) (*DecryptResponse, error) { return nil, status.Errorf(codes.Unimplemented, "method Decrypt not implemented") } diff --git a/keyservice/keyservice.proto b/keyservice/keyservice.proto index a6f5e436f..1d91a5709 100644 --- a/keyservice/keyservice.proto +++ b/keyservice/keyservice.proto @@ -7,6 +7,7 @@ message Key { GcpKmsKey gcp_kms_key = 3; AzureKeyVaultKey azure_keyvault_key = 4; VaultKey vault_key = 5; + AgeKey age_key = 6; } } @@ -37,6 +38,10 @@ message AzureKeyVaultKey { string version = 3; } +message AgeKey { + string recipient = 1; +} + message EncryptRequest { Key key = 1; bytes plaintext = 2; diff --git a/keyservice/server.go b/keyservice/server.go index d2734bd97..08249ff24 100644 --- a/keyservice/server.go +++ b/keyservice/server.go @@ -3,6 +3,7 @@ package keyservice import ( "fmt" + "go.mozilla.org/sops/v3/age" "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/gcpkms" "go.mozilla.org/sops/v3/hcvault" @@ -75,6 +76,18 @@ func (ks *Server) encryptWithVault(key *VaultKey, plaintext []byte) ([]byte, err return []byte(vaultKey.EncryptedKey), nil } +func (ks *Server) encryptWithAge(key *AgeKey, plaintext []byte) ([]byte, error) { + ageKey := age.MasterKey{ + Recipient: key.Recipient, + } + + if err := ageKey.Encrypt(plaintext); err != nil { + return nil, err + } + + return []byte(ageKey.EncryptedKey), nil +} + func (ks *Server) decryptWithPgp(key *PgpKey, ciphertext []byte) ([]byte, error) { pgpKey := pgp.NewMasterKeyFromFingerprint(key.Fingerprint) pgpKey.EncryptedKey = string(ciphertext) @@ -120,6 +133,15 @@ func (ks *Server) decryptWithVault(key *VaultKey, ciphertext []byte) ([]byte, er return []byte(plaintext), err } +func (ks *Server) decryptWithAge(key *AgeKey, ciphertext []byte) ([]byte, error) { + ageKey := age.MasterKey{ + Recipient: key.Recipient, + } + ageKey.EncryptedKey = string(ciphertext) + plaintext, err := ageKey.Decrypt() + return []byte(plaintext), err +} + // Encrypt takes an encrypt request and encrypts the provided plaintext with the provided key, returning the encrypted // result func (ks Server) Encrypt(ctx context.Context, @@ -167,6 +189,14 @@ func (ks Server) Encrypt(ctx context.Context, response = &EncryptResponse{ Ciphertext: ciphertext, } + case *Key_AgeKey: + ciphertext, err := ks.encryptWithAge(k.AgeKey, req.Plaintext) + if err != nil { + return nil, err + } + response = &EncryptResponse{ + Ciphertext: ciphertext, + } case nil: return nil, status.Errorf(codes.NotFound, "Must provide a key") default: @@ -261,6 +291,14 @@ func (ks Server) Decrypt(ctx context.Context, response = &DecryptResponse{ Plaintext: plaintext, } + case *Key_AgeKey: + plaintext, err := ks.decryptWithAge(k.AgeKey, req.Ciphertext) + if err != nil { + return nil, err + } + response = &DecryptResponse{ + Plaintext: plaintext, + } case nil: return nil, grpc.Errorf(codes.NotFound, "Must provide a key") default: diff --git a/kms/keysource.go b/kms/keysource.go index 9fa8172e8..a602aac25 100644 --- a/kms/keysource.go +++ b/kms/keysource.go @@ -63,14 +63,14 @@ func (key *MasterKey) Encrypt(dataKey []byte) error { sess, err := key.createSession() if err != nil { log.WithField("arn", key.Arn).Info("Encryption failed") - return fmt.Errorf("Failed to create session: %v", err) + return fmt.Errorf("Failed to create session: %w", err) } kmsSvc = kms.New(sess) } out, err := kmsSvc.Encrypt(&kms.EncryptInput{Plaintext: dataKey, KeyId: &key.Arn, EncryptionContext: key.EncryptionContext}) if err != nil { log.WithField("arn", key.Arn).Info("Encryption failed") - return fmt.Errorf("Failed to call KMS encryption service: %v", err) + return fmt.Errorf("Failed to call KMS encryption service: %w", err) } key.EncryptedKey = base64.StdEncoding.EncodeToString(out.CiphertextBlob) log.WithField("arn", key.Arn).Info("Encryption succeeded") @@ -98,14 +98,14 @@ func (key *MasterKey) Decrypt() ([]byte, error) { sess, err := key.createSession() if err != nil { log.WithField("arn", key.Arn).Info("Decryption failed") - return nil, fmt.Errorf("Error creating AWS session: %v", err) + return nil, fmt.Errorf("Error creating AWS session: %w", err) } kmsSvc = kms.New(sess) } decrypted, err := kmsSvc.Decrypt(&kms.DecryptInput{CiphertextBlob: k, EncryptionContext: key.EncryptionContext}) if err != nil { log.WithField("arn", key.Arn).Info("Decryption failed") - return nil, fmt.Errorf("Error decrypting key: %v", err) + return nil, fmt.Errorf("Error decrypting key: %w", err) } log.WithField("arn", key.Arn).Info("Decryption succeeded") return decrypted.Plaintext, nil @@ -167,7 +167,7 @@ func (key MasterKey) createStsSession(config aws.Config, sess *session.Session) } stsRoleSessionNameRe, err := regexp.Compile("[^a-zA-Z0-9=,.@-]+") if err != nil { - return nil, fmt.Errorf("Failed to compile STS role session name regex: %v", err) + return nil, fmt.Errorf("Failed to compile STS role session name regex: %w", err) } sanitizedHostname := stsRoleSessionNameRe.ReplaceAllString(hostname, "") stsService := sts.New(sess) @@ -175,13 +175,13 @@ func (key MasterKey) createStsSession(config aws.Config, sess *session.Session) out, err := stsService.AssumeRole(&sts.AssumeRoleInput{ RoleArn: &key.Role, RoleSessionName: &name}) if err != nil { - return nil, fmt.Errorf("Failed to assume role %q: %v", key.Role, err) + return nil, fmt.Errorf("Failed to assume role %q: %w", key.Role, err) } config.Credentials = credentials.NewStaticCredentials(*out.Credentials.AccessKeyId, *out.Credentials.SecretAccessKey, *out.Credentials.SessionToken) sess, err = session.NewSession(&config) if err != nil { - return nil, fmt.Errorf("Failed to create new aws session: %v", err) + return nil, fmt.Errorf("Failed to create new aws session: %w", err) } return sess, nil } diff --git a/pgp/keysource.go b/pgp/keysource.go index fd092339c..f025c66a2 100644 --- a/pgp/keysource.go +++ b/pgp/keysource.go @@ -80,14 +80,14 @@ func (key *MasterKey) encryptWithGPGBinary(dataKey []byte) error { cmd.Stderr = &stderr err := cmd.Run() if err != nil { - return err + return fmt.Errorf("gpg binary failed with error: %s, %s", err, stderr.String()) } key.EncryptedKey = stdout.String() return nil } func getKeyFromKeyServer(fingerprint string) (openpgp.Entity, error) { - log.Warn("Deprecation Warning: GPG key fetching from a keyserver witihin sops will be removed in a future version of sops. See https://github.com/mozilla/sops/issues/727 for more information.") + log.Warn("Deprecation Warning: GPG key fetching from a keyserver within sops will be removed in a future version of sops. See https://github.com/mozilla/sops/issues/727 for more information.") url := fmt.Sprintf("https://keys.openpgp.org/vks/v1/by-fingerprint/%s", fingerprint) resp, err := http.Get(url) diff --git a/pgp/keysource_test.go b/pgp/keysource_test.go index ff9abb7fd..0c10996d1 100644 --- a/pgp/keysource_test.go +++ b/pgp/keysource_test.go @@ -15,12 +15,12 @@ func TestPGP(t *testing.T) { return true } if err := key.Encrypt(x); err != nil { - t.Errorf("Failed to encrypt: %#v err: %v", x, err) + t.Errorf("Failed to encrypt: %#v err: %w", x, err) return false } k, err := key.Decrypt() if err != nil { - t.Errorf("Failed to decrypt: %#v err: %v", x, err) + t.Errorf("Failed to decrypt: %#v err: %w", x, err) return false } return bytes.Equal(x, k) diff --git a/shamir/shamir.go b/shamir/shamir.go index e44da6120..10d7bc3ba 100644 --- a/shamir/shamir.go +++ b/shamir/shamir.go @@ -215,7 +215,7 @@ func Split(secret []byte, parts, threshold int) ([][]byte, error) { // This polynomial crosses the y axis at `val`. p, err := makePolynomial(val, uint8(threshold-1)) if err != nil { - return nil, fmt.Errorf("failed to generate polynomial: %v", err) + return nil, fmt.Errorf("failed to generate polynomial: %w", err) } // Generate a `parts` number of (x,y) pairs diff --git a/sops.go b/sops.go index a41f4a2d1..a6caa0df8 100644 --- a/sops.go +++ b/sops.go @@ -596,7 +596,7 @@ func (m *Metadata) UpdateMasterKeysWithKeyServices(dataKey []byte, svcs []keyser Plaintext: part, }) if err != nil { - keyErrs = append(keyErrs, fmt.Errorf("failed to encrypt new data key with master key %q: %v", key.ToString(), err)) + keyErrs = append(keyErrs, fmt.Errorf("failed to encrypt new data key with master key %q: %w", key.ToString(), err)) continue } key.SetEncryptedDataKey(rsp.Ciphertext) diff --git a/stores/ini/store.go b/stores/ini/store.go index fce92597a..df5405294 100644 --- a/stores/ini/store.go +++ b/stores/ini/store.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" - "reflect" "strconv" "strings" @@ -252,56 +251,6 @@ func (store *Store) encodeMetadataToIniBranch(md stores.Metadata) (sops.TreeBran return branch, nil } -func encodeMetadataItem(prefix string, kind reflect.Kind, field reflect.Value) (map[string]interface{}, error) { - - result := make(map[string]interface{}, 0) - - switch kind { - case reflect.Slice: - slf := field - for j := 0; j < slf.Len(); j++ { - item := slf.Index(j) - p := fmt.Sprintf("%s[%d]", prefix, j) - r, err := encodeMetadataItem(p, item.Type().Kind(), item) - if err != nil { - return result, err - } - for k, v := range r { - result[k] = v - } - } - case reflect.Struct: - for i := 0; i < field.NumField(); i++ { - sf := field.Type().Field(i) - var name string - if prefix == "" { - name = sf.Name - } else { - name = fmt.Sprintf("%s.%s", prefix, sf.Name) - } - r, err := encodeMetadataItem(name, sf.Type.Kind(), field.Field(i)) - if err != nil { - return result, err - } - for k, v := range r { - result[k] = v - } - } - case reflect.Int: - if field.Int() != 0 { - result[prefix] = string(field.Int()) - } - case reflect.String: - if field.String() != "" { - result[prefix] = strings.Replace(field.String(), "\n", "\\n", -1) - } - default: - return result, fmt.Errorf("Cannot encode %s, unexpected type %s", prefix, kind) - } - - return result, nil -} - // EmitPlainFile returns the plaintext INI file bytes corresponding to a sops.TreeBranches object func (store *Store) EmitPlainFile(in sops.TreeBranches) ([]byte, error) { out, err := store.iniFromTreeBranches(in) diff --git a/stores/stores.go b/stores/stores.go index 236940fb2..da8781ab2 100644 --- a/stores/stores.go +++ b/stores/stores.go @@ -15,6 +15,7 @@ import ( "fmt" "go.mozilla.org/sops/v3" + "go.mozilla.org/sops/v3/age" "go.mozilla.org/sops/v3/azkv" "go.mozilla.org/sops/v3/gcpkms" "go.mozilla.org/sops/v3/hcvault" @@ -42,6 +43,7 @@ type Metadata struct { GCPKMSKeys []gcpkmskey `yaml:"gcp_kms" json:"gcp_kms"` AzureKeyVaultKeys []azkvkey `yaml:"azure_kv" json:"azure_kv"` VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"` + AgeKeys []agekey `yaml:"age" json:"age"` LastModified string `yaml:"lastmodified" json:"lastmodified"` MessageAuthenticationCode string `yaml:"mac" json:"mac"` PGPKeys []pgpkey `yaml:"pgp" json:"pgp"` @@ -58,6 +60,7 @@ type keygroup struct { GCPKMSKeys []gcpkmskey `yaml:"gcp_kms,omitempty" json:"gcp_kms,omitempty"` AzureKeyVaultKeys []azkvkey `yaml:"azure_kv,omitempty" json:"azure_kv,omitempty"` VaultKeys []vaultkey `yaml:"hc_vault" json:"hc_vault"` + AgeKeys []agekey `yaml:"age" json:"age"` } type pgpkey struct { @@ -97,6 +100,11 @@ type azkvkey struct { EncryptedDataKey string `yaml:"enc" json:"enc"` } +type agekey struct { + Recipient string `yaml:"recipient" json:"recipient"` + EncryptedDataKey string `yaml:"enc" json:"enc"` +} + // MetadataFromInternal converts an internal SOPS metadata representation to a representation appropriate for storage func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { var m Metadata @@ -115,6 +123,7 @@ func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { m.GCPKMSKeys = gcpkmsKeysFromGroup(group) m.VaultKeys = vaultKeysFromGroup(group) m.AzureKeyVaultKeys = azkvKeysFromGroup(group) + m.AgeKeys = ageKeysFromGroup(group) } else { for _, group := range sopsMetadata.KeyGroups { m.KeyGroups = append(m.KeyGroups, keygroup{ @@ -123,6 +132,7 @@ func MetadataFromInternal(sopsMetadata sops.Metadata) Metadata { GCPKMSKeys: gcpkmsKeysFromGroup(group), VaultKeys: vaultKeysFromGroup(group), AzureKeyVaultKeys: azkvKeysFromGroup(group), + AgeKeys: ageKeysFromGroup(group), }) } } @@ -206,6 +216,19 @@ func azkvKeysFromGroup(group sops.KeyGroup) (keys []azkvkey) { return } +func ageKeysFromGroup(group sops.KeyGroup) (keys []agekey) { + for _, key := range group { + switch key := key.(type) { + case *age.MasterKey: + keys = append(keys, agekey{ + Recipient: key.Recipient, + EncryptedDataKey: key.EncryptedKey, + }) + } + } + return +} + // ToInternal converts a storage-appropriate Metadata struct to a SOPS internal representation func (m *Metadata) ToInternal() (sops.Metadata, error) { lastModified, err := time.Parse(time.RFC3339, m.LastModified) @@ -251,7 +274,7 @@ func (m *Metadata) ToInternal() (sops.Metadata, error) { }, nil } -func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmskey, azkvKeys []azkvkey, vaultKeys []vaultkey) (sops.KeyGroup, error) { +func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmskey, azkvKeys []azkvkey, vaultKeys []vaultkey, ageKeys []agekey) (sops.KeyGroup, error) { var internalGroup sops.KeyGroup for _, kmsKey := range kmsKeys { k, err := kmsKey.toInternal() @@ -288,13 +311,20 @@ func internalGroupFrom(kmsKeys []kmskey, pgpKeys []pgpkey, gcpKmsKeys []gcpkmske } internalGroup = append(internalGroup, k) } + for _, ageKey := range ageKeys { + k, err := ageKey.toInternal() + if err != nil { + return nil, err + } + internalGroup = append(internalGroup, k) + } return internalGroup, nil } func (m *Metadata) internalKeygroups() ([]sops.KeyGroup, error) { var internalGroups []sops.KeyGroup - if len(m.PGPKeys) > 0 || len(m.KMSKeys) > 0 || len(m.GCPKMSKeys) > 0 || len(m.AzureKeyVaultKeys) > 0 || len(m.VaultKeys) > 0 { - internalGroup, err := internalGroupFrom(m.KMSKeys, m.PGPKeys, m.GCPKMSKeys, m.AzureKeyVaultKeys, m.VaultKeys) + if len(m.PGPKeys) > 0 || len(m.KMSKeys) > 0 || len(m.GCPKMSKeys) > 0 || len(m.AzureKeyVaultKeys) > 0 || len(m.VaultKeys) > 0 || len(m.AgeKeys) > 0 { + internalGroup, err := internalGroupFrom(m.KMSKeys, m.PGPKeys, m.GCPKMSKeys, m.AzureKeyVaultKeys, m.VaultKeys, m.AgeKeys) if err != nil { return nil, err } @@ -302,7 +332,7 @@ func (m *Metadata) internalKeygroups() ([]sops.KeyGroup, error) { return internalGroups, nil } else if len(m.KeyGroups) > 0 { for _, group := range m.KeyGroups { - internalGroup, err := internalGroupFrom(group.KMSKeys, group.PGPKeys, group.GCPKMSKeys, group.AzureKeyVaultKeys, group.VaultKeys) + internalGroup, err := internalGroupFrom(group.KMSKeys, group.PGPKeys, group.GCPKMSKeys, group.AzureKeyVaultKeys, group.VaultKeys, group.AgeKeys) if err != nil { return nil, err } @@ -381,6 +411,13 @@ func (pgpKey *pgpkey) toInternal() (*pgp.MasterKey, error) { }, nil } +func (ageKey *agekey) toInternal() (*age.MasterKey, error) { + return &age.MasterKey{ + EncryptedKey: ageKey.EncryptedDataKey, + Recipient: ageKey.Recipient, + }, nil +} + // ExampleComplexTree is an example sops.Tree object exhibiting complex relationships var ExampleComplexTree = sops.Tree{ Branches: sops.TreeBranches{ diff --git a/stores/yaml/store.go b/stores/yaml/store.go index 989a4f056..a3ae2fb46 100644 --- a/stores/yaml/store.go +++ b/stores/yaml/store.go @@ -1,9 +1,12 @@ package yaml //import "go.mozilla.org/sops/v3/stores/yaml" import ( + "bytes" "fmt" + "io" + "strings" - "github.com/mozilla-services/yaml" + "gopkg.in/yaml.v3" "go.mozilla.org/sops/v3" "go.mozilla.org/sops/v3/stores" ) @@ -12,102 +15,232 @@ import ( type Store struct { } -func (store Store) mapSliceToTreeBranch(in yaml.MapSlice) sops.TreeBranch { - branch := make(sops.TreeBranch, 0) - for _, item := range in { - if comment, ok := item.Key.(yaml.Comment); ok { - // Convert the yaml comment to a generic sops comment - branch = append(branch, sops.TreeItem{ - Key: sops.Comment{ - Value: comment.Value, - }, - Value: nil, - }) - } else { - branch = append(branch, sops.TreeItem{ - Key: item.Key, - Value: store.yamlValueToTreeValue(item.Value), - }) +func (store Store) appendCommentToList(comment string, list []interface{}) []interface{} { + if comment != "" { + for _, commentLine := range strings.Split(comment, "\n") { + if commentLine != "" { + list = append(list, sops.Comment{ + Value: commentLine[1:], + }) + } + } + } + return list +} + +func (store Store) appendCommentToMap(comment string, branch sops.TreeBranch) sops.TreeBranch { + if comment != "" { + for _, commentLine := range strings.Split(comment, "\n") { + if commentLine != "" { + branch = append(branch, sops.TreeItem{ + Key: sops.Comment{ + Value: commentLine[1:], + }, + Value: nil, + }) + } } } return branch } -func (store Store) yamlValueToTreeValue(in interface{}) interface{} { - switch in := in.(type) { - case map[interface{}]interface{}: - return store.yamlMapToTreeBranch(in) - case yaml.MapSlice: - return store.mapSliceToTreeBranch(in) - case []interface{}: - return store.yamlSliceToTreeValue(in) - case yaml.Comment: - return sops.Comment{Value: in.Value} - default: - return in +func (store Store) nodeToTreeValue(node *yaml.Node, commentsWereHandled bool) (interface{}, error) { + switch node.Kind { + case yaml.DocumentNode: + panic("documents should never be passed here") + case yaml.SequenceNode: + var result []interface{} + if !commentsWereHandled { + result = store.appendCommentToList(node.HeadComment, result) + result = store.appendCommentToList(node.LineComment, result) + } + for _, item := range node.Content { + result = store.appendCommentToList(item.HeadComment, result) + result = store.appendCommentToList(item.LineComment, result) + val, err := store.nodeToTreeValue(item, true) + if err != nil { + return nil, err + } + result = append(result, val) + result = store.appendCommentToList(item.FootComment, result) + } + if !commentsWereHandled { + result = store.appendCommentToList(node.FootComment, result) + } + return result, nil + case yaml.MappingNode: + branch := make(sops.TreeBranch, 0) + return store.appendYamlNodeToTreeBranch(node, branch, false) + case yaml.ScalarNode: + var result interface{} + node.Decode(&result) + return result, nil + case yaml.AliasNode: + return store.nodeToTreeValue(node.Alias, false); } + return nil, nil } -func (store *Store) yamlSliceToTreeValue(in []interface{}) []interface{} { - for i, v := range in { - in[i] = store.yamlValueToTreeValue(v) +func (store Store) appendYamlNodeToTreeBranch(node *yaml.Node, branch sops.TreeBranch, commentsWereHandled bool) (sops.TreeBranch, error) { + var err error + if !commentsWereHandled { + branch = store.appendCommentToMap(node.HeadComment, branch) + branch = store.appendCommentToMap(node.LineComment, branch) + } + switch node.Kind { + case yaml.DocumentNode: + for _, item := range node.Content { + branch, err = store.appendYamlNodeToTreeBranch(item, branch, false) + if err != nil { + return nil, err + } + } + case yaml.SequenceNode: + return nil, fmt.Errorf("YAML documents that are sequences are not supported") + case yaml.MappingNode: + for i := 0; i < len(node.Content); i += 2 { + key := node.Content[i] + value := node.Content[i + 1] + branch = store.appendCommentToMap(key.HeadComment, branch) + branch = store.appendCommentToMap(key.LineComment, branch) + handleValueComments := value.Kind == yaml.ScalarNode || value.Kind == yaml.AliasNode + if handleValueComments { + branch = store.appendCommentToMap(value.HeadComment, branch) + branch = store.appendCommentToMap(value.LineComment, branch) + } + var keyValue interface{} + key.Decode(&keyValue) + valueTV, err := store.nodeToTreeValue(value, handleValueComments) + if err != nil { + return nil, err + } + branch = append(branch, sops.TreeItem{ + Key: keyValue, + Value: valueTV, + }) + if handleValueComments { + branch = store.appendCommentToMap(value.FootComment, branch) + } + branch = store.appendCommentToMap(key.FootComment, branch) + } + case yaml.ScalarNode: + // A empty document with a document start marker without comments results in null + if node.ShortTag() == "!!null" { + return branch, nil + } + return nil, fmt.Errorf("YAML documents that are values are not supported") + case yaml.AliasNode: + branch, err = store.appendYamlNodeToTreeBranch(node.Alias, branch, false) } - return in + if !commentsWereHandled { + branch = store.appendCommentToMap(node.FootComment, branch) + } + return branch, nil } -func (store *Store) yamlMapToTreeBranch(in map[interface{}]interface{}) sops.TreeBranch { +func (store Store) yamlDocumentNodeToTreeBranch(in yaml.Node) (sops.TreeBranch, error) { branch := make(sops.TreeBranch, 0) - for k, v := range in { - branch = append(branch, sops.TreeItem{ - Key: k.(string), - Value: store.yamlValueToTreeValue(v), - }) + return store.appendYamlNodeToTreeBranch(&in, branch, false) +} + +func (store *Store) addCommentsHead(node *yaml.Node, comments []string) []string { + if len(comments) > 0 { + comment := "#" + strings.Join(comments, "\n#") + if len(node.HeadComment) > 0 { + node.HeadComment = comment + "\n" + node.HeadComment + } else { + node.HeadComment = comment + } } - return branch + return nil } -func (store Store) treeValueToYamlValue(in interface{}) interface{} { +func (store *Store) addCommentsFoot(node *yaml.Node, comments []string) []string { + if len(comments) > 0 { + comment := "#" + strings.Join(comments, "\n#") + if len(node.FootComment) > 0 { + node.FootComment += "\n" + comment + } else { + node.FootComment = comment + } + } + return nil +} + +func (store *Store) treeValueToNode(in interface{}) *yaml.Node { switch in := in.(type) { case sops.TreeBranch: - return store.treeBranchToYamlMap(in) - case sops.Comment: - return yaml.Comment{in.Value} + var mapping = &yaml.Node{} + mapping.Kind = yaml.MappingNode + store.appendTreeBranch(in, mapping) + return mapping case []interface{}: - var out []interface{} - for _, v := range in { - out = append(out, store.treeValueToYamlValue(v)) - } - return out + var sequence = &yaml.Node{} + sequence.Kind = yaml.SequenceNode + store.appendSequence(in, sequence) + return sequence default: - return in + var valueNode = &yaml.Node{} + valueNode.Encode(in) + return valueNode } } -func (store Store) treeBranchToYamlMap(in sops.TreeBranch) yaml.MapSlice { - branch := make(yaml.MapSlice, 0) +func (store *Store) appendSequence(in []interface{}, sequence *yaml.Node) { + var comments []string + var beginning bool = true for _, item := range in { + if comment, ok := item.(sops.Comment); ok { + comments = append(comments, comment.Value) + } else { + if beginning { + comments = store.addCommentsHead(sequence, comments) + beginning = false + } + itemNode := store.treeValueToNode(item) + comments = store.addCommentsHead(itemNode, comments) + sequence.Content = append(sequence.Content, itemNode) + } + } + if len(comments) > 0 { + if beginning { + comments = store.addCommentsHead(sequence, comments) + } else { + comments = store.addCommentsFoot(sequence.Content[len(sequence.Content) - 1], comments) + } + } +} + +func (store *Store) appendTreeBranch(branch sops.TreeBranch, mapping *yaml.Node) { + var comments []string + var beginning bool = true + for _, item := range branch { if comment, ok := item.Key.(sops.Comment); ok { - branch = append(branch, yaml.MapItem{ - Key: store.treeValueToYamlValue(comment), - Value: nil, - }) + comments = append(comments, comment.Value) } else { - branch = append(branch, yaml.MapItem{ - Key: item.Key, - Value: store.treeValueToYamlValue(item.Value), - }) + if beginning { + comments = store.addCommentsHead(mapping, comments) + beginning = false + } + var keyNode = &yaml.Node{} + keyNode.Encode(item.Key) + comments = store.addCommentsHead(keyNode, comments) + valueNode := store.treeValueToNode(item.Value) + mapping.Content = append(mapping.Content, keyNode, valueNode) + } + } + if len(comments) > 0 { + if beginning { + comments = store.addCommentsHead(mapping, comments) + } else { + comments = store.addCommentsFoot(mapping.Content[len(mapping.Content) - 1], comments) } } - return branch } // LoadEncryptedFile loads the contents of an encrypted yaml file onto a // sops.Tree runtime object func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { - var data []yaml.MapSlice - if err := (yaml.CommentUnmarshaler{}).UnmarshalDocuments(in, &data); err != nil { - return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err) - } // Because we don't know what fields the input file will have, we have to // load the file in two steps. // First, we load the file's metadata, the structure of which is known. @@ -123,14 +256,33 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { if err != nil { return sops.Tree{}, err } + var data yaml.Node + if err := yaml.Unmarshal(in, &data); err != nil { + return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err) + } var branches sops.TreeBranches - for _, doc := range data { - for i, item := range doc { - if item.Key == "sops" { // Erase - doc = append(doc[:i], doc[i+1:]...) + d := yaml.NewDecoder(bytes.NewReader(in)) + for true { + var data yaml.Node + err := d.Decode(&data) + if err == io.EOF { + break + } + if err != nil { + return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err) + } + + branch, err := store.yamlDocumentNodeToTreeBranch(data) + if err != nil { + return sops.Tree{}, fmt.Errorf("Error unmarshaling input YAML: %s", err) + } + + for i, elt := range branch { + if elt.Key == "sops" { // Erase + branch = append(branch[:i], branch[i+1:]...) } } - branches = append(branches, store.mapSliceToTreeBranch(doc)) + branches = append(branches, branch) } return sops.Tree{ Branches: branches, @@ -139,16 +291,25 @@ func (store *Store) LoadEncryptedFile(in []byte) (sops.Tree, error) { } // LoadPlainFile loads the contents of a plaintext yaml file onto a -// sops.Tree runtime obejct +// sops.Tree runtime object func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { - var data []yaml.MapSlice - if err := (yaml.CommentUnmarshaler{}).UnmarshalDocuments(in, &data); err != nil { - return nil, fmt.Errorf("Error unmarshaling input YAML: %s", err) - } - var branches sops.TreeBranches - for _, doc := range data { - branches = append(branches, store.mapSliceToTreeBranch(doc)) + d := yaml.NewDecoder(bytes.NewReader(in)) + for true { + var data yaml.Node + err := d.Decode(&data) + if err == io.EOF { + break + } + if err != nil { + return nil, fmt.Errorf("Error unmarshaling input YAML: %s", err) + } + + branch, err := store.yamlDocumentNodeToTreeBranch(data) + if err != nil { + return nil, fmt.Errorf("Error unmarshaling input YAML: %s", err) + } + branches = append(branches, branch) } return branches, nil } @@ -156,45 +317,70 @@ func (store *Store) LoadPlainFile(in []byte) (sops.TreeBranches, error) { // EmitEncryptedFile returns the encrypted bytes of the yaml file corresponding to a // sops.Tree runtime object func (store *Store) EmitEncryptedFile(in sops.Tree) ([]byte, error) { - out := []byte{} - for i, branch := range in.Branches { - if i > 0 { - out = append(out, "---\n"...) - } - yamlMap := store.treeBranchToYamlMap(branch) - yamlMap = append(yamlMap, yaml.MapItem{Key: "sops", Value: stores.MetadataFromInternal(in.Metadata)}) - tout, err := (&yaml.YAMLMarshaler{Indent: 4}).Marshal(yamlMap) + var b bytes.Buffer + e := yaml.NewEncoder(io.Writer(&b)) + e.SetIndent(4) + for _, branch := range in.Branches { + // Document root + var doc = yaml.Node{} + doc.Kind = yaml.DocumentNode + // Add global mapping + var mapping = yaml.Node{} + mapping.Kind = yaml.MappingNode + doc.Content = append(doc.Content, &mapping) + // Create copy of branch with metadata appended + branch = append(sops.TreeBranch(nil), branch...) + branch = append(branch, sops.TreeItem{ + Key: "sops", + Value: stores.MetadataFromInternal(in.Metadata), + }) + // Marshal branch to global mapping node + store.appendTreeBranch(branch, &mapping) + // Encode YAML + err := e.Encode(&doc) if err != nil { return nil, fmt.Errorf("Error marshaling to yaml: %s", err) } - out = append(out, tout...) } - return out, nil + e.Close() + return b.Bytes(), nil } // EmitPlainFile returns the plaintext bytes of the yaml file corresponding to a // sops.TreeBranches runtime object func (store *Store) EmitPlainFile(branches sops.TreeBranches) ([]byte, error) { - var out []byte - for i, branch := range branches { - if i > 0 { - out = append(out, "---\n"...) + var b bytes.Buffer + e := yaml.NewEncoder(io.Writer(&b)) + e.SetIndent(4) + for _, branch := range branches { + // Document root + var doc = yaml.Node{} + doc.Kind = yaml.DocumentNode + // Add global mapping + var mapping = yaml.Node{} + mapping.Kind = yaml.MappingNode + // Marshal branch to global mapping node + store.appendTreeBranch(branch, &mapping) + if len(mapping.Content) == 0 { + doc.HeadComment = mapping.HeadComment + } else { + doc.Content = append(doc.Content, &mapping) } - yamlMap := store.treeBranchToYamlMap(branch) - tmpout, err := (&yaml.YAMLMarshaler{Indent: 4}).Marshal(yamlMap) + // Encode YAML + err := e.Encode(&doc) if err != nil { return nil, fmt.Errorf("Error marshaling to yaml: %s", err) } - out = append(out[:], tmpout[:]...) } - return out, nil + e.Close() + return b.Bytes(), nil } // EmitValue returns bytes corresponding to a single encoded value // in a generic interface{} object func (store *Store) EmitValue(v interface{}) ([]byte, error) { - v = store.treeValueToYamlValue(v) - return (&yaml.YAMLMarshaler{Indent: 4}).Marshal(v) + n := store.treeValueToNode(v) + return yaml.Marshal(n) } // EmitExample returns the bytes corresponding to an example complex tree diff --git a/stores/yaml/store_test.go b/stores/yaml/store_test.go index bed993507..be6b90eb2 100644 --- a/stores/yaml/store_test.go +++ b/stores/yaml/store_test.go @@ -15,8 +15,18 @@ key1_a: value --- key2: value2`) +var PLAIN_0 = []byte(`# comment 0 +key1: value +key1_a: value +# ^ comment 1 +`) + var BRANCHES = sops.TreeBranches{ sops.TreeBranch{ + sops.TreeItem{ + Key: sops.Comment{" comment 0"}, + Value: nil, + }, sops.TreeItem{ Key: "key1", Value: "value", @@ -38,6 +48,49 @@ var BRANCHES = sops.TreeBranches{ }, } +var COMMENT_1 = []byte(`# test +a: + b: null + # foo +`) + +var COMMENT_2 = []byte(`a: + # foo + b: null +`) + +var COMMENT_3_IN = []byte(`## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + + jobLabel: node-exporter + extraArgs: + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ +`) +var COMMENT_3_OUT = []byte(`## Configuration for prometheus-node-exporter subchart +## +prometheus-node-exporter: + podLabels: + ## Add the 'node-exporter' label to be used by serviceMonitor to match standard common usage in rules and grafana dashboards + ## + jobLabel: node-exporter + extraArgs: + - --collector.filesystem.ignored-mount-points=^/(dev|proc|sys|var/lib/docker/.+)($|/) + - --collector.filesystem.ignored-fs-types=^(autofs|binfmt_misc|cgroup|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|mqueue|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|sysfs|tracefs)$ +`) + +var COMMENT_4 = []byte(`# foo +`) + +var COMMENT_5 = []byte(`# foo +--- +key: value +`) + func TestUnmarshalMetadataFromNonSOPSFile(t *testing.T) { data := []byte(`hello: 2`) _, err := (&Store{}).LoadEncryptedFile(data) @@ -49,3 +102,86 @@ func TestLoadPlainFile(t *testing.T) { assert.Nil(t, err) assert.Equal(t, BRANCHES, branches) } + +func TestComment1(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile(COMMENT_1) + assert.Nil(t, err) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(COMMENT_1), string(bytes)) + assert.Equal(t, COMMENT_1, bytes) +} + +func TestComment2(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile(COMMENT_2) + assert.Nil(t, err) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(COMMENT_2), string(bytes)) + assert.Equal(t, COMMENT_2, bytes) +} + +func TestComment3(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile(COMMENT_3_IN) + assert.Nil(t, err) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(COMMENT_3_OUT), string(bytes)) + assert.Equal(t, COMMENT_3_OUT, bytes) +} + +/* TODO: re-enable once https://github.com/go-yaml/yaml/pull/690 is merged +func TestComment4(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile(COMMENT_4) + assert.Nil(t, err) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(COMMENT_4), string(bytes)) + assert.Equal(t, COMMENT_4, bytes) +} + +func TestComment5(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile(COMMENT_5) + assert.Nil(t, err) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, string(COMMENT_5), string(bytes)) + assert.Equal(t, COMMENT_5, bytes) +} +*/ + +func TestEmpty(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile([]byte(``)) + assert.Nil(t, err) + assert.Equal(t, len(branches), 0) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, ``, string(bytes)) +} + +/* TODO: re-enable once https://github.com/go-yaml/yaml/pull/690 is merged +func TestEmpty2(t *testing.T) { + // First iteration: load and store + branches, err := (&Store{}).LoadPlainFile([]byte(`---`)) + assert.Nil(t, err) + assert.Equal(t, len(branches), 1) + assert.Equal(t, len(branches[0]), 0) + bytes, err := (&Store{}).EmitPlainFile(branches) + assert.Nil(t, err) + assert.Equal(t, ``, string(bytes)) +} +*/ + +func TestEmitValue(t *testing.T) { + // First iteration: load and store + bytes, err := (&Store{}).EmitValue(BRANCHES[0]) + assert.Nil(t, err) + assert.Equal(t, string(PLAIN_0), string(bytes)) + assert.Equal(t, PLAIN_0, bytes) +} diff --git a/version/version.go b/version/version.go index 892ff6f59..cbcff60f0 100644 --- a/version/version.go +++ b/version/version.go @@ -11,7 +11,7 @@ import ( ) // Version represents the value of the current semantic version -const Version = "3.6.1" +const Version = "3.7.0" // PrintVersion handles the version command for sops func PrintVersion(c *cli.Context) { @@ -74,7 +74,7 @@ func RetrieveLatestVersionFromUpstream() (string, error) { // try to parse the version as semver _, err := semver.Make(comps[1]) if err != nil { - return "", fmt.Errorf("Retrieved version %q does not match semver format: %v", comps[1], err) + return "", fmt.Errorf("Retrieved version %q does not match semver format: %w", comps[1], err) } return comps[1], nil }