Merge pull request #1056 from mozilla/develop

v3.7.3

.circleci/config.yml

@@ -18,15 +18,22 @@ jobs:
     resource_class: large
     steps:
       - checkout
-      - setup_remote_docker
+      - setup_remote_docker:
+          version: 20.10.11
       - run:
           name: Build containers
           command: |
             docker build -t mozilla/sops .
             docker tag mozilla/sops "mozilla/sops:$CIRCLE_SHA1"
+      - run:
+          name: Build containers (alpine)
+          command: |
+            # Just to ensure the container can be built.
+            docker build -f Dockerfile.alpine -t mozilla/sops:alpine .
 
   push:
-    machine: true
+    machine:
+      image: ubuntu-2004:202111-02
     resource_class: large
     steps:
       - checkout

.github/workflows/release.yml

@@ -34,6 +34,8 @@ jobs:
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set RELEASE_NUMBER
         run: echo "RELEASE_NUMBER=$(echo $RELEASE_VERSION | cut -c2-)" >> $GITHUB_ENV
+      - name: Build linux amd64 binary
+        run: GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64 go.mozilla.org/sops/v3/cmd/sops && cp dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64 dist/sops-${{ env.RELEASE_VERSION }}.linux
       - name: Build linux arm64 binary
         run: GOOS=linux GOARCH=arm64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.linux.arm64 go.mozilla.org/sops/v3/cmd/sops
       - name: Build darwin amd64 binary
@@ -44,8 +46,6 @@ jobs:
         run: GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin.arm64 go.mozilla.org/sops/v3/cmd/sops
       - name: Build windows binary
         run: GOOS=windows CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.exe go.mozilla.org/sops/v3/cmd/sops
-      - name: Copy already built linux binary
-        run: cp tmppkg/usr/local/bin/sops dist/sops-${{ env.RELEASE_VERSION }}.linux && cp tmppkg/usr/local/bin/sops dist/sops-${{ env.RELEASE_VERSION }}.linux.amd64
       - name: Create release
         uses: "mozilla/action-automatic-releases@latest"
         with:

CHANGELOG.rst

@@ -1,6 +1,22 @@
 Changelog
 =========
 
+3.7.3
+-----
+Changes:
+
+* Upgrade dependencies (#1024, #1045)
+* Build alpine container in CI (#1018, #1032, #1025)
+* keyservice: accept KeyServiceServer in LocalClient (#1035)
+* Add support for GCP Service Account within `GOOGLE_CREDENTIALS` (#953)
+
+Bug fixes:
+
+* Upload the correct binary for the linux amd64 build (#1026)
+* Fix bug when specifying multiple age recipients (#966)
+* Allow for empty yaml maps (#908)
+* Limit AWS role names to 64 characters (#1037)
+
 3.7.2
 -----
 Changes:

Dockerfile.alpine

@@ -1,4 +1,3 @@
-FROM golang:1.12-alpine3.10 AS builder
 FROM golang:1.17-alpine3.15 AS builder
 
 RUN apk --no-cache add make

Makefile

@@ -113,7 +113,7 @@ download-index:
 	bash make_download_page.sh
 
 mock:
-	go get github.com/vektra/mockery/.../
+	go install github.com/vektra/mockery/.../
 	mockery -dir vendor/github.com/aws/aws-sdk-go/service/kms/kmsiface/ -name KMSAPI -output kms/mocks
 
 .PHONY: all test generate clean vendor functional-tests mock

age/keys.txt

@@ -1,3 +1,6 @@
 # created: 2020-07-18T03:16:47-07:00
 # public key: age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw
 AGE-SECRET-KEY-1NJT5YCS2LWU4V4QAJQ6R4JNU7LXPDX602DZ9NUFANVU5GDTGUWCQ5T59M6
+# created: 2021-12-12T01:39:30+01:00
+# public key: age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep
+AGE-SECRET-KEY-1T0Z66WSXS6RMNCPSL7P2E8N4Q7SUD8VMG9ND27S08JL7Y2XAU9EQECHDS7

age/keysource.go

@@ -184,7 +184,7 @@ func MasterKeysFromRecipients(commaSeparatedRecipients string) ([]*MasterKey, er
 	var keys []*MasterKey
 
 	for _, recipient := range recipients {
-		key, err := MasterKeyFromRecipient(recipient)
+		key, err := masterKeyFromRecipient(recipient)
 
 		if err != nil {
 			return nil, err
@@ -196,8 +196,8 @@ func MasterKeysFromRecipients(commaSeparatedRecipients string) ([]*MasterKey, er
 	return keys, nil
 }
 
-// MasterKeyFromRecipient takes a Bech32-encoded public key and returns a new MasterKey.
-func MasterKeyFromRecipient(recipient string) (*MasterKey, error) {
+// masterKeyFromRecipient takes a Bech32-encoded public key and returns a new MasterKey.
+func masterKeyFromRecipient(recipient string) (*MasterKey, error) {
 	recipient = strings.TrimSpace(recipient)
 	parsedRecipient, err := parseRecipient(recipient)
 

age/keysource_test.go

@@ -21,76 +21,131 @@ func TestMasterKeysFromRecipientsEmpty(t *testing.T) {
 	assert.Equal(recipients, make([]*MasterKey, 0))
 }
 
-func TestMasterKeyFromRecipientWithLeadingAndTrailingSpaces(t *testing.T) {
+func TestMasterKeyFromRecipientWithLeadingAndTrailingSpacesSingle(t *testing.T) {
 	assert := assert.New(t)
 
-	key, err := MasterKeyFromRecipient("  age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw  ")
+	commaSeparatedRecipients := "  age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw  "
+	keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)
 
 	assert.NoError(err)
 
-	assert.Equal(key.Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	assert.Equal(len(keys), 1)
+	assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
 }
 
-func TestAge(t *testing.T) {
+func TestMasterKeyFromRecipientWithLeadingAndTrailingSpacesMultiple(t *testing.T) {
 	assert := assert.New(t)
 
-	key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	commaSeparatedRecipients := "  age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw  ,  age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep  "
+	keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)
 
 	assert.NoError(err)
-	assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString())
 
-	dataKey := []byte("abcdefghijklmnopqrstuvwxyz123456")
+	assert.Equal(len(keys), 2)
+	assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")
+}
+
+func TestMasterKeysFromRecipientsWithSingle(t *testing.T) {
+	assert := assert.New(t)
+
+	commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw"
+	keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)
 
-	err = key.Encrypt(dataKey)
 	assert.NoError(err)
 
-	_, filename, _, _ := runtime.Caller(0)
-	err = os.Setenv(SopsAgeKeyFileEnv, path.Join(path.Dir(filename), "keys.txt"))
+	assert.Equal(len(keys), 1)
+	assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+}
+
+func TestMasterKeysFromRecipientsWithMultiple(t *testing.T) {
+	assert := assert.New(t)
+
+	commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
+	keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)
+
 	assert.NoError(err)
 
-	decryptedKey, err := key.Decrypt()
+	assert.Equal(len(keys), 2)
+	assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")
+}
+
+func TestAge(t *testing.T) {
+	assert := assert.New(t)
+
+	commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
+	keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)
+
 	assert.NoError(err)
-	assert.Equal(dataKey, decryptedKey)
+
+	assert.Equal(len(keys), 2)
+	assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")
+
+	dataKey := []byte("abcdefghijklmnopqrstuvwxyz123456")
+
+	for _, key := range keys {
+		err = key.Encrypt(dataKey)
+		assert.NoError(err)
+
+		_, filename, _, _ := runtime.Caller(0)
+		err = os.Setenv("SOPS_AGE_KEY_FILE", path.Join(path.Dir(filename), "keys.txt"))
+		assert.NoError(err)
+
+		decryptedKey, err := key.Decrypt()
+		assert.NoError(err)
+		assert.Equal(dataKey, decryptedKey)
+	}
+
 }
 
 func TestAgeDotEnv(t *testing.T) {
 	assert := assert.New(t)
 
-	key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
+	keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)
 
 	assert.NoError(err)
-	assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString())
+
+	assert.Equal(len(keys), 2)
+	assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")
 
 	dotenv := `IMAGE_PREFIX=repo/service-
 APPLICATION_KEY=K6pfAWuUVND9Fz5SC7jmA6pfAWuUVND9Fz5SC7jmA
 KEY_ID=003683d721f2ae683d721f2a1
 DOMAIN=files.127.0.0.1.nip.io`
 	dataKey := []byte(dotenv)
 
-	err = key.Encrypt(dataKey)
+	err = keys[0].Encrypt(dataKey)
 	assert.NoError(err)
 
 	_, filename, _, _ := runtime.Caller(0)
 	err = os.Setenv(SopsAgeKeyFileEnv, path.Join(path.Dir(filename), "keys.txt"))
 	defer os.Unsetenv(SopsAgeKeyFileEnv)
 	assert.NoError(err)
 
-	decryptedKey, err := key.Decrypt()
+	decryptedKey, err := keys[0].Decrypt()
 	assert.NoError(err)
 	assert.Equal(dataKey, decryptedKey)
 }
 
 func TestAgeEnv(t *testing.T) {
 	assert := assert.New(t)
 
-	key, err := MasterKeyFromRecipient("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	commaSeparatedRecipients := "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw,age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep"
+	keys, err := MasterKeysFromRecipients(commaSeparatedRecipients)
 
 	assert.NoError(err)
-	assert.Equal("age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw", key.ToString())
+
+	assert.Equal(len(keys), 2)
+	assert.Equal(keys[0].Recipient, "age1yt3tfqlfrwdwx0z0ynwplcr6qxcxfaqycuprpmy89nr83ltx74tqdpszlw")
+	assert.Equal(keys[1].Recipient, "age1tmaae3ld5vpevmsh5yacsauzx8jetg300mpvc4ugp5zr5l6ssq9sla97ep")
 
 	dataKey := []byte("abcdefghijklmnopqrstuvwxyz123456")
 
-	err = key.Encrypt(dataKey)
+	err = keys[0].Encrypt(dataKey)
 	assert.NoError(err)
 
 	_, filename, _, _ := runtime.Caller(0)
@@ -100,7 +155,7 @@ func TestAgeEnv(t *testing.T) {
 	defer os.Unsetenv(SopsAgeKeyEnv)
 	assert.NoError(err)
 
-	decryptedKey, err := key.Decrypt()
+	decryptedKey, err := keys[0].Decrypt()
 	assert.NoError(err)
 	assert.Equal(dataKey, decryptedKey)
 }

cmd/sops/main.go

@@ -442,12 +442,14 @@ func main() {
 							group = append(group, k)
 						}
 						for _, recipient := range ageRecipients {
-							k, err := age.MasterKeyFromRecipient(recipient)
+							keys, err := age.MasterKeysFromRecipients(recipient)
 							if err != nil {
 								log.WithError(err).Error("Failed to add key")
 								continue
 							}
-							group = append(group, k)
+							for _, key := range keys {
+								group = append(group, key)
+							}
 						}
 						return groups.Add(groups.AddOpts{
 							InputPath:      c.String("file"),

config/config.go

@@ -153,11 +153,13 @@ func getKeyGroupsFromCreationRule(cRule *creationRule, kmsEncryptionContext map[
 		for _, group := range cRule.KeyGroups {
 			var keyGroup sops.KeyGroup
 			for _, k := range group.Age {
-				key, err := age.MasterKeyFromRecipient(k)
+				keys, err := age.MasterKeysFromRecipients(k)
 				if err != nil {
 					return nil, err
 				}
-				keyGroup = append(keyGroup, key)
+				for _, key := range keys {
+					keyGroup = append(keyGroup, key)
+				}
 			}
 			for _, k := range group.PGP {
 				keyGroup = append(keyGroup, pgp.NewMasterKeyFromFingerprint(k))

gcpkms/keysource.go

@@ -3,16 +3,16 @@ package gcpkms //import "go.mozilla.org/sops/v3/gcpkms"
 import (
 	"encoding/base64"
 	"fmt"
+	"google.golang.org/api/option"
+	"os"
 	"regexp"
 	"strings"
 	"time"
 
 	"go.mozilla.org/sops/v3/logging"
 
-	"golang.org/x/net/context"
-	"golang.org/x/oauth2/google"
-
 	"github.com/sirupsen/logrus"
+	"golang.org/x/net/context"
 	cloudkms "google.golang.org/api/cloudkms/v1"
 )
 
@@ -131,12 +131,15 @@ func (key MasterKey) createCloudKMSService() (*cloudkms.Service, error) {
 	}
 
 	ctx := context.Background()
-	client, err := google.DefaultClient(ctx, cloudkms.CloudPlatformScope)
-	if err != nil {
+	var options []option.ClientOption
+
+	if credentials, err := getGoogleCredentials(); err != nil {
 		return nil, err
+	} else if len(credentials) > 0 {
+		options = append(options, option.WithCredentialsJSON(credentials))
 	}
 
-	cloudkmsService, err := cloudkms.New(client)
+	cloudkmsService, err := cloudkms.NewService(ctx, options...)
 	if err != nil {
 		return nil, err
 	}
@@ -151,3 +154,16 @@ func (key MasterKey) ToMap() map[string]interface{} {
 	out["created_at"] = key.CreationDate.UTC().Format(time.RFC3339)
 	return out
 }
+
+// getGoogleCredentials looks for a GCP Service Account in the environment
+// variable: GOOGLE_CREDENTIALS, set as either a path to a credentials file or directly as the
+// variable's value in JSON format.
+//
+// If not set, will default to use GOOGLE_APPLICATION_CREDENTIALS
+func getGoogleCredentials() ([]byte, error) {
+	defaultCredentials := os.Getenv("GOOGLE_CREDENTIALS")
+	if _, err := os.Stat(defaultCredentials); err == nil {
+		return os.ReadFile(defaultCredentials)
+	}
+	return []byte(defaultCredentials), nil
+}