Merge branch 'develop' into sops-age-key-env

.circleci/config.yml

@@ -4,10 +4,10 @@ workflows:
   build-and-deploy:
     jobs:
       - build
-      - push:     
+      - push:
           filters:
             tags:
-              only: /^v.*/   
+              only: /^v.*/
             branches:
               ignore: /.*/
 jobs:
@@ -30,7 +30,7 @@ jobs:
     resource_class: large
     steps:
       - checkout
-      - run: 
+      - run:
           name: semver check
           command: |
             MAJOR=$(echo ${CIRCLE_TAG#v} | cut -d"." -f1)
@@ -46,8 +46,8 @@ jobs:
 
             This job uses the semver from the git TAG as the public version to publish.
 
-            - This should only run on workflows triggered by a tag. 
-            - The tag name should be a semver like 'v1.2.3' 
+            - This should only run on workflows triggered by a tag.
+            - The tag name should be a semver like 'v1.2.3'
             - The version should follow conventions documented at https://github.com/fsaintjacques/semver-tool
             EOF
             exit 1

.github/workflows/cli.yml

@@ -12,22 +12,26 @@ on:
 
 jobs:
   build:
-    name: Build and test ${{ matrix.os }}
+    name: Build and test ${{ matrix.os }} ${{ matrix.arch }}
     runs-on: ubuntu-latest
     strategy:
       matrix:
         os: [linux, darwin, windows]
+        arch: [amd64, arm64]
+        exclude:
+          - os: windows
+            arch: arm64
     env:
       VAULT_VERSION: "1.1.3"
       VAULT_TOKEN: "root"
       VAULT_ADDR: "http://127.0.0.1:8200"
     steps:
       - name: Install dependencies
         run: sudo apt-get update && sudo apt-get install git -y
-      - name: Set up Go 1.13
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: 1.13
+          go-version: 1.17
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
@@ -37,13 +41,24 @@ jobs:
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ runner.os }}-go-
-      - name: Build
+      - name: Build Linux and Darwin
+        if: matrix.os != 'windows'
+        run: GOOS=${{ matrix.os }} GOARCH=${{ matrix.arch }} go build -o sops-${{ matrix.os }}-${{ matrix.arch }}-${{ github.sha }} -v ./cmd/sops
+      - name: Build Windows
+        if: matrix.os == 'windows'
         run: GOOS=${{ matrix.os }} go build -o sops-${{ matrix.os }}-${{ github.sha }} -v ./cmd/sops
       - name: Import test GPG keys
         run: for i in 1 2 3 4 5; do gpg --import pgp/sops_functional_tests_key.asc && break || sleep 15; done
       - name: Test
         run: make test
-      - name: Upload artifact
+      - name: Upload artifact for Linux and Darwin
+        if: matrix.os != 'windows'
+        uses: actions/upload-artifact@v2
+        with:
+          name: sops-${{ matrix.os }}-${{ matrix.arch }}-${{ github.sha }}
+          path: sops-${{ matrix.os }}-${{ matrix.arch }}-${{ github.sha }}
+      - name: Upload artifact for Windows
+        if: matrix.os == 'windows'
         uses: actions/upload-artifact@v2
         with:
           name: sops-${{ matrix.os }}-${{ github.sha }}
@@ -63,9 +78,9 @@ jobs:
         uses: actions/checkout@v2
       - uses: actions/download-artifact@v2
         with:
-          name: sops-linux-${{ github.sha }}
+          name: sops-linux-amd64-${{ github.sha }}
       - name: Move SOPS binary
-        run: mv sops-linux-${{ github.sha }} ./functional-tests/sops
+        run: mv sops-linux-amd64-${{ github.sha }} ./functional-tests/sops
       - name: Make SOPS binary executable
         run: chmod +x ./functional-tests/sops
       - name: Download Vault

.github/workflows/release.yml

@@ -15,13 +15,15 @@ jobs:
         run: sudo apt-get update && sudo apt-get install git ruby rpm -y
       - name: Install fpm
         run: gem install fpm || sudo gem install fpm
-      - name: Set up Go 1.15
+      - name: Set up Go 1.17
         uses: actions/setup-go@v2
         with:
-          go-version: 1.15
+          go-version: 1.17
         id: go
       - name: Check out code into the Go module directory
         uses: actions/checkout@v2
+      - name: Go vendor
+        run: go mod vendor
       - name: Make release directory
         run: mkdir dist
       - name: Build deb and rpm
@@ -32,8 +34,10 @@ jobs:
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
       - name: Set RELEASE_NUMBER
         run: echo "RELEASE_NUMBER=$(echo $RELEASE_VERSION | cut -c2-)" >> $GITHUB_ENV
-      - name: Build darwin binary
-        run: GOOS=darwin CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin go.mozilla.org/sops/v3/cmd/sops
+      - name: Build darwin amd64 binary
+        run: GOOS=darwin GOARCH=amd64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin.amd64 go.mozilla.org/sops/v3/cmd/sops
+      - name: Build darwin arm64 binary
+        run: GOOS=darwin GOARCH=arm64 CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.darwin.arm64 go.mozilla.org/sops/v3/cmd/sops
       - name: Build windows binary
         run: GOOS=windows CGO_ENABLED=0 go build -mod vendor -o dist/sops-${{ env.RELEASE_VERSION }}.exe go.mozilla.org/sops/v3/cmd/sops
       - name: Copy already built linux binary
@@ -45,7 +49,8 @@ jobs:
           prerelease: true
           files: |
             dist/sops-${{ env.RELEASE_VERSION }}.exe
-            dist/sops-${{ env.RELEASE_VERSION }}.darwin
+            dist/sops-${{ env.RELEASE_VERSION }}.darwin.amd64
+            dist/sops-${{ env.RELEASE_VERSION }}.darwin.arm64
             dist/sops-${{ env.RELEASE_VERSION }}.linux
             dist/sops_${{ env.RELEASE_NUMBER }}_amd64.deb
             dist/sops-${{ env.RELEASE_NUMBER }}-1.x86_64.rpm

Dockerfile

@@ -1,10 +1,10 @@
-FROM golang:1.15
+FROM golang:1.17
 
 COPY . /go/src/go.mozilla.org/sops
 WORKDIR /go/src/go.mozilla.org/sops
 
 RUN CGO_ENABLED=1 make install
 RUN apt-get update
-RUN apt-get install -y vim python-pip emacs
+RUN apt-get install -y vim python3-pip emacs
 RUN pip install awscli
 ENV EDITOR vim

Dockerfile.alpine

@@ -1,4 +1,5 @@
 FROM golang:1.12-alpine3.10 AS builder
+FROM golang:1.17-alpine3.15 AS builder
 
 RUN apk --no-cache add make
 
@@ -8,7 +9,7 @@ WORKDIR /go/src/go.mozilla.org/sops
 RUN CGO_ENABLED=1 make install
 
 
-FROM alpine:3.10
+FROM alpine:3.15
 
 RUN apk --no-cache add \
   vim ca-certificates

Makefile

@@ -3,7 +3,7 @@
 # file, You can obtain one at http://mozilla.org/MPL/2.0/.
 
 PROJECT		:= go.mozilla.org/sops/v3
-GO 		:= GO15VENDOREXPERIMENT=1 GO111MODULE=on GOPROXY=https://proxy.golang.org go
+GO 		:= GOPROXY=https://proxy.golang.org go
 GOLINT 		:= golint
 
 all: test vet generate install functional-tests

README.rst

@@ -706,12 +706,6 @@ By default, ``sops`` uses the key server ``keys.openpgp.org`` to retrieve the GP
 keys that are not present in the local keyring.
 This is no longer configurable. You can learn more about why from this write-up: `SKS Keyserver Network Under Attack <https://gist.github.com/rjhansen/67ab921ffb4084c865b3618d6955275f>`_.
 
-Example: place the following in your ``~/.bashrc``
-
-.. code:: bash
-
-	SOPS_GPG_KEYSERVER = 'gpg.example.com'
-
 
 Key groups
 ~~~~~~~~~~
@@ -1209,7 +1203,7 @@ This file will not work in sops:
 	  - array
 	  - elements
 
-But this one will because because the ``sops`` key can be added at the same level as the
+But this one will work because the ``sops`` key can be added at the same level as the
 ``data`` key.
 
 .. code:: yaml

age/keysource.go

@@ -20,7 +20,6 @@ func init() {
 	log = logging.NewLogger("AGE")
 }
 
-const privateKeySizeLimit = 1 << 24 // 16 MiB
 const SopsAgeKeyEnv = "SOPS_AGE_KEY"
 const SopsAgeKeyFileEnv = "SOPS_AGE_KEY_FILE"
 
@@ -30,7 +29,6 @@ type MasterKey struct {
 	Recipient    string // a Bech32-encoded public key
 	EncryptedKey string // a sops data key encrypted with age
 
-	parsedIdentity  *age.X25519Identity  // a parsed age private key
 	parsedRecipient *age.X25519Recipient // a parsed age public key
 }
 

example.yaml

@@ -25,7 +25,7 @@ this:
 somelist_unencrypted:
 - all elements of this list
 - remain in clear text
-- because of the _encrypted suffix in the key
+- because of the _unencrypted suffix in the key
 nested_unencrypted:
     this:
         is:

go.mod

@@ -1,42 +1,28 @@
 module go.mozilla.org/sops/v3
 
-go 1.13
+go 1.17
 
 require (
 	cloud.google.com/go v0.43.0
 	filippo.io/age v1.0.0-beta7
 	github.com/Azure/azure-sdk-for-go v31.2.0+incompatible
-	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
 	github.com/Azure/go-autorest/autorest v0.9.0
 	github.com/Azure/go-autorest/autorest/azure/auth v0.1.0
-	github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect
-	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
-	github.com/Microsoft/go-winio v0.4.14 // indirect
-	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
 	github.com/aws/aws-sdk-go v1.37.18
 	github.com/blang/semver v3.5.1+incompatible
-	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
-	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
-	github.com/docker/go-connections v0.4.0 // indirect
-	github.com/docker/go-units v0.4.0 // indirect
 	github.com/fatih/color v1.7.0
 	github.com/golang/protobuf v1.4.1
 	github.com/google/go-cmp v0.5.0
 	github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf
-	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
 	github.com/goware/prefixer v0.0.0-20160118172347-395022866408
 	github.com/hashicorp/vault/api v1.0.4
 	github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c
 	github.com/lib/pq v1.2.0
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/go-wordwrap v1.0.0
-	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
-	github.com/opencontainers/image-spec v1.0.1 // indirect
-	github.com/opencontainers/runc v0.1.1 // indirect
 	github.com/ory/dockertest v3.3.4+incompatible
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.4.2
-	github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect
 	github.com/stretchr/testify v1.5.1
 	go.mozilla.org/gopgagent v0.0.0-20170926210634-4d7ea76ff71a
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83
@@ -49,5 +35,59 @@ require (
 	gopkg.in/ini.v1 v1.44.0
 	gopkg.in/urfave/cli.v1 v1.20.0
 	gopkg.in/yaml.v3 v3.0.0-20210107172259-749611fa9fcc
+)
+
+require (
+	github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect
+	github.com/Azure/go-autorest/autorest/adal v0.5.0 // indirect
+	github.com/Azure/go-autorest/autorest/azure/cli v0.1.0 // indirect
+	github.com/Azure/go-autorest/autorest/date v0.1.0 // indirect
+	github.com/Azure/go-autorest/autorest/to v0.3.0 // indirect
+	github.com/Azure/go-autorest/autorest/validation v0.2.0 // indirect
+	github.com/Azure/go-autorest/logger v0.1.0 // indirect
+	github.com/Azure/go-autorest/tracing v0.5.0 // indirect
+	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
+	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
+	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/dimchansky/utfbom v1.1.0 // indirect
+	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/docker/go-units v0.4.0 // indirect
+	github.com/golang/snappy v0.0.1 // indirect
+	github.com/googleapis/gax-go/v2 v2.0.5 // indirect
+	github.com/gotestyourself/gotestyourself v2.2.0+incompatible // indirect
+	github.com/hashicorp/errwrap v1.0.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.1 // indirect
+	github.com/hashicorp/go-multierror v1.0.0 // indirect
+	github.com/hashicorp/go-retryablehttp v0.5.4 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.1 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.1 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/vault/sdk v0.1.13 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.1 // indirect
+	github.com/mattn/go-colorable v0.0.9 // indirect
+	github.com/mattn/go-isatty v0.0.3 // indirect
+	github.com/mitchellh/mapstructure v1.1.2 // indirect
+	github.com/opencontainers/go-digest v1.0.0-rc1 // indirect
+	github.com/opencontainers/image-spec v1.0.1 // indirect
+	github.com/opencontainers/runc v0.1.1 // indirect
+	github.com/pierrec/lz4 v2.0.5+incompatible // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/sergi/go-diff v1.1.0 // indirect
+	github.com/smartystreets/goconvey v0.0.0-20190710185942-9d28bd7c0945 // indirect
+	github.com/stretchr/objx v0.1.1 // indirect
+	go.opencensus.io v0.22.0 // indirect
+	golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 // indirect
+	golang.org/x/text v0.3.3 // indirect
+	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
+	google.golang.org/appengine v1.6.1 // indirect
+	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 // indirect
+	gopkg.in/square/go-jose.v2 v2.3.1 // indirect
+	gopkg.in/yaml.v2 v2.2.8 // indirect
 	gotest.tools v2.2.0+incompatible // indirect
 )

hcvault/keysource_test.go

@@ -25,7 +25,8 @@ func TestMain(m *testing.M) {
 		logger.Fatalf("Could not start resource: %s", err)
 	}
 
-	os.Setenv("VAULT_ADDR", fmt.Sprintf("http://127.0.0.1:%v", resource.GetPort("8200/tcp")))
+	vaultAddr := fmt.Sprintf("http://%s", resource.GetHostPort("8200/tcp"))
+	os.Setenv("VAULT_ADDR", vaultAddr)
 	os.Setenv("VAULT_TOKEN", "secret")
 	// exponential backoff-retry, because the application in the container might not be ready to accept connections yet
 	if err := pool.Retry(func() error {
@@ -45,7 +46,7 @@ func TestMain(m *testing.M) {
 		logger.Fatalf("Could not connect to docker: %s", err)
 	}
 
-	key := NewMasterKey(fmt.Sprintf("http://127.0.0.1:%v", resource.GetPort("8200/tcp")), "sops", "main")
+	key := NewMasterKey(vaultAddr, "sops", "main")
 	err = key.createVaultTransitAndKey()
 	if err != nil {
 		logger.Fatal(err)