Merge pull request #3763 from souleb/feat-ignore

Add the possibility to ignore files with build and diff Kustomization

cmd/flux/build_kustomization.go

@@ -46,14 +46,22 @@ flux build kustomization my-app --path ./path/to/local/manifests --kustomization
 
 # Build in dry-run mode without connecting to the cluster.
 # Note that variable substitutions from Secrets and ConfigMaps are skipped in dry-run mode.
-flux build kustomization my-app --path ./path/to/local/manifests --kustomization-file ./path/to/local/my-app.yaml --dry-run`,
+flux build kustomization my-app --path ./path/to/local/manifests \
+	--kustomization-file ./path/to/local/my-app.yaml \
+	--dry-run
+
+# Exclude files by providing a comma separated list of entries that follow the .gitignore pattern fromat.
+flux build kustomization my-app --path ./path/to/local/manifests \
+	--kustomization-file ./path/to/local/my-app.yaml \
+	--ignore-paths "/to_ignore/**/*.yaml,ignore.yaml"`,
 	ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)),
 	RunE:              buildKsCmdRun,
 }
 
 type buildKsFlags struct {
 	kustomizationFile string
 	path              string
+	ignorePaths       []string
 	dryRun            bool
 }
 
@@ -62,6 +70,7 @@ var buildKsArgs buildKsFlags
 func init() {
 	buildKsCmd.Flags().StringVar(&buildKsArgs.path, "path", "", "Path to the manifests location.")
 	buildKsCmd.Flags().StringVar(&buildKsArgs.kustomizationFile, "kustomization-file", "", "Path to the Flux Kustomization YAML file.")
+	buildKsCmd.Flags().StringSliceVar(&buildKsArgs.ignorePaths, "ignore-paths", nil, "set paths to ignore in .gitignore format")
 	buildKsCmd.Flags().BoolVar(&buildKsArgs.dryRun, "dry-run", false, "Dry run mode.")
 	buildCmd.AddCommand(buildKsCmd)
 }
@@ -97,12 +106,14 @@ func buildKsCmdRun(cmd *cobra.Command, args []string) (err error) {
 			build.WithKustomizationFile(buildKsArgs.kustomizationFile),
 			build.WithDryRun(buildKsArgs.dryRun),
 			build.WithNamespace(*kubeconfigArgs.Namespace),
+			build.WithIgnore(buildKsArgs.ignorePaths),
 		)
 	} else {
 		builder, err = build.NewBuilder(name, buildKsArgs.path,
 			build.WithClientConfig(kubeconfigArgs, kubeclientOptions),
 			build.WithTimeout(rootArgs.timeout),
 			build.WithKustomizationFile(buildKsArgs.kustomizationFile),
+			build.WithIgnore(buildKsArgs.ignorePaths),
 		)
 	}
 

cmd/flux/build_kustomization_test.go

@@ -63,6 +63,12 @@ func TestBuildKustomization(t *testing.T) {
 			resultFile: "./testdata/build-kustomization/podinfo-with-var-substitution-result.yaml",
 			assertFunc: "assertGoldenTemplateFile",
 		},
+		{
+			name:       "build ignore",
+			args:       "build kustomization podinfo --path ./testdata/build-kustomization/ignore --ignore-paths \"!configmap.yaml,!secret.yaml\"",
+			resultFile: "./testdata/build-kustomization/podinfo-with-ignore-result.yaml",
+			assertFunc: "assertGoldenTemplateFile",
+		},
 	}
 
 	tmpl := map[string]string{

cmd/flux/diff_kustomization.go

@@ -37,14 +37,21 @@ Exit status: 0 No differences were found. 1 Differences were found. >1 diff fail
 flux diff kustomization my-app --path ./path/to/local/manifests
 
 # Preview using a local flux kustomization file
-flux diff kustomization my-app --path ./path/to/local/manifests --kustomization-file ./path/to/local/my-app.yaml`,
+flux diff kustomization my-app --path ./path/to/local/manifests \
+	--kustomization-file ./path/to/local/my-app.yaml
+
+# Exclude files by providing a comma separated list of entries that follow the .gitignore pattern fromat.
+flux diff kustomization my-app --path ./path/to/local/manifests \
+	--kustomization-file ./path/to/local/my-app.yaml \
+	--ignore-paths "/to_ignore/**/*.yaml,ignore.yaml"`,
 	ValidArgsFunction: resourceNamesCompletionFunc(kustomizev1.GroupVersion.WithKind(kustomizev1.KustomizationKind)),
 	RunE:              diffKsCmdRun,
 }
 
 type diffKsFlags struct {
 	kustomizationFile string
 	path              string
+	ignorePaths       []string
 	progressBar       bool
 }
 
@@ -53,6 +60,7 @@ var diffKsArgs diffKsFlags
 func init() {
 	diffKsCmd.Flags().StringVar(&diffKsArgs.path, "path", "", "Path to a local directory that matches the specified Kustomization.spec.path.")
 	diffKsCmd.Flags().BoolVar(&diffKsArgs.progressBar, "progress-bar", true, "Boolean to set the progress bar. The default value is true.")
+	diffKsCmd.Flags().StringSliceVar(&diffKsArgs.ignorePaths, "ignore-paths", nil, "set paths to ignore in .gitignore format")
 	diffKsCmd.Flags().StringVar(&diffKsArgs.kustomizationFile, "kustomization-file", "", "Path to the Flux Kustomization YAML file.")
 	diffCmd.AddCommand(diffKsCmd)
 }
@@ -86,12 +94,16 @@ func diffKsCmdRun(cmd *cobra.Command, args []string) error {
 			build.WithClientConfig(kubeconfigArgs, kubeclientOptions),
 			build.WithTimeout(rootArgs.timeout),
 			build.WithKustomizationFile(diffKsArgs.kustomizationFile),
-			build.WithProgressBar())
+			build.WithProgressBar(),
+			build.WithIgnore(diffKsArgs.ignorePaths),
+		)
 	} else {
 		builder, err = build.NewBuilder(name, diffKsArgs.path,
 			build.WithClientConfig(kubeconfigArgs, kubeclientOptions),
 			build.WithTimeout(rootArgs.timeout),
-			build.WithKustomizationFile(diffKsArgs.kustomizationFile))
+			build.WithKustomizationFile(diffKsArgs.kustomizationFile),
+			build.WithIgnore(diffKsArgs.ignorePaths),
+		)
 	}
 
 	if err != nil {

cmd/flux/testdata/build-kustomization/ignore/.sourceignore

@@ -0,0 +1,2 @@
+# exclude all
+/*

cmd/flux/testdata/build-kustomization/ignore/configmap.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+data:
+  var: test
+kind: ConfigMap
+metadata:
+  name: configmap_ignore

cmd/flux/testdata/build-kustomization/ignore/not_deployable/ignore_svc.yaml

@@ -0,0 +1,17 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: do_not_deploy
+spec:
+  type: ClusterIP
+  selector:
+    app: podinfo
+  ports:
+    - name: http
+      port: 9898
+      protocol: TCP
+      targetPort: http
+    - port: 9999
+      targetPort: grpc
+      protocol: TCP
+      name: grpc

cmd/flux/testdata/build-kustomization/ignore/secret.yaml

@@ -0,0 +1,7 @@
+apiVersion: v1
+data:
+  token: KipTT1BTKio=
+kind: Secret
+metadata:
+  name: secret_ignore
+type: Opaque

cmd/flux/testdata/build-kustomization/podinfo-with-ignore-result.yaml

@@ -0,0 +1,23 @@
+apiVersion: v1
+data:
+  var: test
+kind: ConfigMap
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: configmap_ignore
+  namespace: default
+---
+apiVersion: v1
+data:
+  token: KipTT1BTKio=
+kind: Secret
+metadata:
+  labels:
+    kustomize.toolkit.fluxcd.io/name: podinfo
+    kustomize.toolkit.fluxcd.io/namespace: {{ .fluxns }}
+  name: secret_ignore
+  namespace: default
+type: Opaque
+---

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/fluxcd/pkg/apis/meta v1.0.0
 	github.com/fluxcd/pkg/git v0.11.0
 	github.com/fluxcd/pkg/git/gogit v0.8.1
-	github.com/fluxcd/pkg/kustomize v1.1.0
+	github.com/fluxcd/pkg/kustomize v1.1.1
 	github.com/fluxcd/pkg/oci v0.22.0
 	github.com/fluxcd/pkg/runtime v0.35.0
 	github.com/fluxcd/pkg/sourceignore v0.3.3

go.sum

@@ -222,8 +222,8 @@ github.com/fluxcd/pkg/git v0.11.0/go.mod h1:VHRVlrZMHNoWBlaSAWxlGH6Vwlb9VRazUhPU
 github.com/fluxcd/pkg/git/gogit v0.8.1 h1:Q3EV2WBX6HiXSmsHyrwFzwl82gO4ZtFwb675iQPWwVc=
 github.com/fluxcd/pkg/git/gogit v0.8.1/go.mod h1:5M27gCl0gyo6l+ht9HwZSzimPY3LahKVIJ7/1vCCctg=
 github.com/fluxcd/pkg/gittestserver v0.8.1 h1:FMqnZBuS/11+9NhtLv9UAg+wm/v0Nf+hHeUOi2wJR3Q=
-github.com/fluxcd/pkg/kustomize v1.1.0 h1:4qoTJBCtlg9RbO6YUwTNV3SttvXRIZxkjRRJE+jbsKk=
-github.com/fluxcd/pkg/kustomize v1.1.0/go.mod h1:i+Z9iPAoSz28oH0FmDI73iqZ3oXZxQR2O3HfhdsWhfo=
+github.com/fluxcd/pkg/kustomize v1.1.1 h1:hYFJGi+fiaecY4gXvx52fumlvDEq/1RdFbaev67oBhE=
+github.com/fluxcd/pkg/kustomize v1.1.1/go.mod h1:i+Z9iPAoSz28oH0FmDI73iqZ3oXZxQR2O3HfhdsWhfo=
 github.com/fluxcd/pkg/oci v0.22.0 h1:6QRvCj1YXGEGXHyVkmKiBvYxsE0sEjUrpFknM513MbQ=
 github.com/fluxcd/pkg/oci v0.22.0/go.mod h1:y0jUgMqb6ionfX+8AjhnoG8D6hSSx4elhtrQ7Uo0WzI=
 github.com/fluxcd/pkg/runtime v0.35.0 h1:9PYLcul8qdfLYQArcYpHe/QuMqyhAGGFN9F7uY/QVX4=

internal/build/build.go

@@ -25,6 +25,7 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"strings"
 	"sync"
 	"time"
 
@@ -71,6 +72,7 @@ type Builder struct {
 	namespace         string
 	resourcesPath     string
 	kustomizationFile string
+	ignore            []string
 	// mu is used to synchronize access to the kustomization file
 	mu            sync.Mutex
 	action        kustomize.Action
@@ -156,6 +158,14 @@ func WithDryRun(dryRun bool) BuilderOptionFunc {
 	}
 }
 
+// WithIgnore sets ignore field
+func WithIgnore(ignore []string) BuilderOptionFunc {
+	return func(b *Builder) error {
+		b.ignore = ignore
+		return nil
+	}
+}
+
 // NewBuilder returns a new Builder
 // It takes a kustomization name and a path to the resources
 // It also takes a list of BuilderOptionFunc to configure the builder
@@ -326,9 +336,13 @@ func (b *Builder) generate(kustomization kustomizev1.Kustomization, dirPath stri
 	if err != nil {
 		return "", err
 	}
-	gen := kustomize.NewGeneratorWithIgnore("", "", unstructured.Unstructured{Object: data})
 
-	// acuire the lock
+	// a scanner will be used down the line to parse the list
+	// so we have to make sure to unclude newlines
+	ignoreList := strings.Join(b.ignore, "\n")
+	gen := kustomize.NewGeneratorWithIgnore("", ignoreList, unstructured.Unstructured{Object: data})
+
+	// acquire the lock
 	b.mu.Lock()
 	defer b.mu.Unlock()