.github/workflows/e2e.yaml

@@ -40,7 +40,7 @@ jobs:
       - name: Setup Go
         uses: actions/setup-go@v3
         with:
-          go-version: 1.17.x
+          go-version: 1.18.x
       - name: Setup Kubernetes
         uses: engineerd/setup-kind@v0.5.0
         with:

CHANGELOG.md

@@ -2,6 +2,27 @@
 
 All notable changes to this project are documented in this file.
 
+## 0.26.3
+
+**Release date:** 2022-07-13
+
+This prerelease adds a retry mechanism for "not found" errors when
+downloading artifacts and recovers from SOPS store panics.
+Some dependencies have also been updated to patch upstream CVEs.
+
+Fixes:
+- decryptor: recover from SOPS store panic
+  [#691](https://github.com/fluxcd/kustomize-controller/pull/691)
+
+Improvements:
+- Retry downloading artifacts on not found errors
+  [#689](https://github.com/fluxcd/kustomize-controller/pull/689)
+- Update dependencies
+  [#692](https://github.com/fluxcd/kustomize-controller/pull/692)
+  [#696](https://github.com/fluxcd/kustomize-controller/pull/696)
+- build: Upgrade to Go 1.18
+  [#694](https://github.com/fluxcd/kustomize-controller/pull/694)
+
 ## 0.26.2
 
 **Release date:** 2022-06-29

DEVELOPMENT.md

@@ -25,7 +25,7 @@ If any of the above dependencies are not present on your system, the first invoc
 ## How to run the test suite
 
 Prerequisites:
-* Go >= 1.17
+* Go >= 1.18
 
 You can run the test suite by simply doing
 

Dockerfile

@@ -1,4 +1,4 @@
-ARG GO_VERSION=1.17
+ARG GO_VERSION=1.18
 ARG XX_VERSION=1.1.0
 
 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx

Makefile

@@ -99,8 +99,8 @@ api-docs: gen-crd-api-reference-docs
 
 # Run go mod tidy
 tidy:
-	cd api; rm -f go.sum; go mod tidy -compat=1.17
-	rm -f go.sum; go mod tidy -compat=1.17
+	cd api; rm -f go.sum; go mod tidy -compat=1.18
+	rm -f go.sum; go mod tidy -compat=1.18
 
 # Run go fmt against code
 fmt:

api/go.mod

@@ -1,6 +1,6 @@
 module github.com/fluxcd/kustomize-controller/api
 
-go 1.17
+go 1.18
 
 require (
 	github.com/fluxcd/pkg/apis/kustomize v0.4.2
@@ -11,7 +11,7 @@ require (
 )
 
 // Fix CVE-2022-28948
-replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.0
+replace gopkg.in/yaml.v3 => gopkg.in/yaml.v3 v3.0.1
 
 require (
 	github.com/go-logr/logr v1.2.2 // indirect

api/go.sum

@@ -878,8 +878,8 @@ gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
 gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
-gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
-gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk=
 gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

config/manager/kustomization.yaml

@@ -5,4 +5,4 @@ resources:
 images:
   - name: fluxcd/kustomize-controller
     newName: fluxcd/kustomize-controller
-    newTag: v0.26.2
+    newTag: v0.26.3

controllers/kustomization_controller.go

@@ -19,19 +19,13 @@ package controllers
 import (
 	"bytes"
 	"context"
-	"crypto/sha1"
-	"crypto/sha256"
 	"fmt"
-	"io"
-	"net/http"
-	"net/url"
 	"os"
 	"sort"
 	"strings"
 	"time"
 
 	securejoin "github.com/cyphar/filepath-securejoin"
-	"github.com/hashicorp/go-retryablehttp"
 	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimeta "k8s.io/apimachinery/pkg/api/meta"
@@ -62,7 +56,6 @@ import (
 	"github.com/fluxcd/pkg/runtime/metrics"
 	"github.com/fluxcd/pkg/runtime/predicates"
 	"github.com/fluxcd/pkg/ssa"
-	"github.com/fluxcd/pkg/untar"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
@@ -79,7 +72,7 @@ import (
 // KustomizationReconciler reconciles a Kustomization object
 type KustomizationReconciler struct {
 	client.Client
-	httpClient            *retryablehttp.Client
+	artifactFetcher       *ArtifactFetcher
 	requeueDependency     time.Duration
 	Scheme                *runtime.Scheme
 	EventRecorder         kuberecorder.EventRecorder
@@ -122,15 +115,7 @@ func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts Kustom
 
 	r.requeueDependency = opts.DependencyRequeueInterval
 	r.statusManager = fmt.Sprintf("gotk-%s", r.ControllerName)
-
-	// Configure the retryable http client used for fetching artifacts.
-	// By default it retries 10 times within a 3.5 minutes window.
-	httpClient := retryablehttp.NewClient()
-	httpClient.RetryWaitMin = 5 * time.Second
-	httpClient.RetryWaitMax = 30 * time.Second
-	httpClient.RetryMax = opts.HTTPRetry
-	httpClient.Logger = nil
-	r.httpClient = httpClient
+	r.artifactFetcher = NewArtifactFetcher(opts.HTTPRetry)
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&kustomizev1.Kustomization{}, builder.WithPredicates(
@@ -268,6 +253,18 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 
 	// reconcile kustomization by applying the latest revision
 	reconciledKustomization, reconcileErr := r.reconcile(ctx, *kustomization.DeepCopy(), source)
+
+	// requeue if the artifact is not found
+	if reconcileErr == ArtifactNotFoundError {
+		msg := fmt.Sprintf("Source is not ready, artifact not found, retrying in %s", r.requeueDependency.String())
+		log.Info(msg)
+		if err := r.patchStatus(ctx, req, kustomizev1.KustomizationProgressing(kustomization, msg).Status); err != nil {
+			log.Error(err, "unable to update status for artifact not found")
+			return ctrl.Result{Requeue: true}, err
+		}
+		return ctrl.Result{RequeueAfter: r.requeueDependency}, nil
+	}
+
 	if err := r.patchStatus(ctx, req, reconciledKustomization.Status); err != nil {
 		return ctrl.Result{Requeue: true}, err
 	}
@@ -320,7 +317,7 @@ func (r *KustomizationReconciler) reconcile(
 	defer os.RemoveAll(tmpDir)
 
 	// download artifact and extract files
-	err = r.download(source.GetArtifact(), tmpDir)
+	err = r.artifactFetcher.Fetch(source.GetArtifact(), tmpDir)
 	if err != nil {
 		return kustomizev1.KustomizationNotReady(
 			kustomization,
@@ -526,70 +523,6 @@ func (r *KustomizationReconciler) checkDependencies(source sourcev1.Source, kust
 	return nil
 }
 
-func (r *KustomizationReconciler) download(artifact *sourcev1.Artifact, tmpDir string) error {
-	artifactURL := artifact.URL
-	if hostname := os.Getenv("SOURCE_CONTROLLER_LOCALHOST"); hostname != "" {
-		u, err := url.Parse(artifactURL)
-		if err != nil {
-			return err
-		}
-		u.Host = hostname
-		artifactURL = u.String()
-	}
-
-	req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil)
-	if err != nil {
-		return fmt.Errorf("failed to create a new request: %w", err)
-	}
-
-	resp, err := r.httpClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to download artifact, error: %w", err)
-	}
-	defer resp.Body.Close()
-
-	// check response
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to download artifact from %s, status: %s", artifactURL, resp.Status)
-	}
-
-	var buf bytes.Buffer
-
-	// verify checksum matches origin
-	if err := r.verifyArtifact(artifact, &buf, resp.Body); err != nil {
-		return err
-	}
-
-	// extract
-	if _, err = untar.Untar(&buf, tmpDir); err != nil {
-		return fmt.Errorf("failed to untar artifact, error: %w", err)
-	}
-
-	return nil
-}
-
-func (r *KustomizationReconciler) verifyArtifact(artifact *sourcev1.Artifact, buf *bytes.Buffer, reader io.Reader) error {
-	hasher := sha256.New()
-
-	// for backwards compatibility with source-controller v0.17.2 and older
-	if len(artifact.Checksum) == 40 {
-		hasher = sha1.New()
-	}
-
-	// compute checksum
-	mw := io.MultiWriter(hasher, buf)
-	if _, err := io.Copy(mw, reader); err != nil {
-		return err
-	}
-
-	if checksum := fmt.Sprintf("%x", hasher.Sum(nil)); checksum != artifact.Checksum {
-		return fmt.Errorf("failed to verify artifact: computed checksum '%s' doesn't match advertised '%s'",
-			checksum, artifact.Checksum)
-	}
-
-	return nil
-}
-
 func (r *KustomizationReconciler) getSource(ctx context.Context, kustomization kustomizev1.Kustomization) (sourcev1.Source, error) {
 	var source sourcev1.Source
 	sourceNamespace := kustomization.GetNamespace()

controllers/kustomization_decryptor.go

@@ -57,7 +57,7 @@ const (
 	// DecryptionProviderSOPS is the SOPS provider name.
 	DecryptionProviderSOPS = "sops"
 	// DecryptionPGPExt is the extension of the file containing an armored PGP
-	//key.
+	// key.
 	DecryptionPGPExt = ".asc"
 	// DecryptionAgeExt is the extension of the file containing an age key
 	// file.
@@ -263,7 +263,16 @@ func (d *KustomizeDecryptor) ImportKeys(ctx context.Context) error {
 // for the input format, gathers the data key for it from the key service,
 // and then decrypts the file data with the retrieved data key.
 // It returns the decrypted bytes in the provided output format, or an error.
-func (d *KustomizeDecryptor) SopsDecryptWithFormat(data []byte, inputFormat, outputFormat formats.Format) ([]byte, error) {
+func (d *KustomizeDecryptor) SopsDecryptWithFormat(data []byte, inputFormat, outputFormat formats.Format) (_ []byte, err error) {
+	defer func() {
+		// It was discovered that malicious input and/or output instructions can
+		// make SOPS panic. Recover from this panic and return as an error.
+		if r := recover(); r != nil {
+			err = fmt.Errorf("failed to emit encrypted %s file as decrypted %s: %v",
+				sopsFormatToString[inputFormat], sopsFormatToString[outputFormat], r)
+		}
+	}()
+
 	store := common.StoreForFormat(inputFormat)
 
 	tree, err := store.LoadEncryptedFile(data)