use workers to limit the paralellism when verifying images

Signed-off-by: cpanato <ctadeu@gmail.com>
sigstore · Jun 22, 2023 · 0d3d5ff · 0d3d5ff
1 parent ff648d5
commit 0d3d5ff
Show file tree

Hide file tree

Showing 16 changed files with 136 additions and 55 deletions.
diff --git a/cmd/cosign/cli/dockerfile.go b/cmd/cosign/cli/dockerfile.go
@@ -16,6 +16,8 @@
 package cli
 
 import (
+	"fmt"
+
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/dockerfile"
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/verify"
 	"github.com/spf13/cobra"
@@ -106,9 +108,15 @@ Shell-like variables in the Dockerfile's FROM lines will be substituted with val
 					Offline:                      o.CommonVerifyOptions.Offline,
 					TSACertChainPath:             o.CommonVerifyOptions.TSACertChainPath,
 					IgnoreTlog:                   o.CommonVerifyOptions.IgnoreTlog,
+					MaxWorkers:                   o.CommonVerifyOptions.MaxWorkers,
 				},
 				BaseOnly: o.BaseImageOnly,
 			}
+
+			if o.CommonVerifyOptions.MaxWorkers == 0 {
+				return fmt.Errorf("please set the --max-worker flag to a value that is greater than 0")
+			}
+
 			return v.Exec(cmd.Context(), args)
 		},
 	}

diff --git a/cmd/cosign/cli/manifest.go b/cmd/cosign/cli/manifest.go
@@ -16,6 +16,8 @@
 package cli
 
 import (
+	"fmt"
+
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/manifest"
 	"github.com/sigstore/cosign/v2/cmd/cosign/cli/verify"
 	"github.com/spf13/cobra"
@@ -101,8 +103,14 @@ against the transparency log.`,
 					Offline:                      o.CommonVerifyOptions.Offline,
 					TSACertChainPath:             o.CommonVerifyOptions.TSACertChainPath,
 					IgnoreTlog:                   o.CommonVerifyOptions.IgnoreTlog,
+					MaxWorkers:                   o.CommonVerifyOptions.MaxWorkers,
 				},
 			}
+
+			if o.CommonVerifyOptions.MaxWorkers == 0 {
+				return fmt.Errorf("please set the --max-worker flag to a value that is greater than 0")
+			}
+
 			return v.Exec(cmd.Context(), args)
 		},
 	}

diff --git a/cmd/cosign/cli/options/verify.go b/cmd/cosign/cli/options/verify.go
@@ -23,6 +23,7 @@ type CommonVerifyOptions struct {
 	Offline          bool // Force offline verification
 	TSACertChainPath string
 	IgnoreTlog       bool
+	MaxWorkers       int
 }
 
 func (o *CommonVerifyOptions) AddFlags(cmd *cobra.Command) {
@@ -36,6 +37,9 @@ func (o *CommonVerifyOptions) AddFlags(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&o.IgnoreTlog, "insecure-ignore-tlog", false,
 		"ignore transparency log verification, to be used when an artifact signature has not been uploaded to the transparency log. Artifacts "+
 			"cannot be publicly verified when not included in a log")
+
+	cmd.Flags().IntVar(&o.MaxWorkers, "max-workers", 10,
+		"the amount of maximum workers for parallel executions")
 }
 
 // VerifyOptions is the top level wrapper for the `verify` command.

diff --git a/cmd/cosign/cli/verify.go b/cmd/cosign/cli/verify.go
@@ -126,6 +126,11 @@ against the transparency log.`,
 				Offline:                      o.CommonVerifyOptions.Offline,
 				TSACertChainPath:             o.CommonVerifyOptions.TSACertChainPath,
 				IgnoreTlog:                   o.CommonVerifyOptions.IgnoreTlog,
+				MaxWorkers:                   o.CommonVerifyOptions.MaxWorkers,
+			}
+
+			if o.CommonVerifyOptions.MaxWorkers == 0 {
+				return fmt.Errorf("please set the --max-worker flag to a value that is greater than 0")
 			}
 
 			if o.Registry.AllowInsecure {
@@ -220,6 +225,11 @@ against the transparency log.`,
 				Offline:                      o.CommonVerifyOptions.Offline,
 				TSACertChainPath:             o.CommonVerifyOptions.TSACertChainPath,
 				IgnoreTlog:                   o.CommonVerifyOptions.IgnoreTlog,
+				MaxWorkers:                   o.CommonVerifyOptions.MaxWorkers,
+			}
+
+			if o.CommonVerifyOptions.MaxWorkers == 0 {
+				return fmt.Errorf("please set the --max-worker flag to a value that is greater than 0")
 			}
 
 			ctx := cmd.Context()

diff --git a/cmd/cosign/cli/verify/verify.go b/cmd/cosign/cli/verify/verify.go
@@ -78,6 +78,7 @@ type VerifyCommand struct {
 	Offline                      bool
 	TSACertChainPath             string
 	IgnoreTlog                   bool
+	MaxWorkers                   int
 }
 
 // Exec runs the verification command
@@ -125,6 +126,7 @@ func (c *VerifyCommand) Exec(ctx context.Context, images []string) (err error) {
 		Identities:                   identities,
 		Offline:                      c.Offline,
 		IgnoreTlog:                   c.IgnoreTlog,
+		MaxWorkers:                   c.MaxWorkers,
 	}
 	if c.CheckClaims {
 		co.ClaimVerifier = cosign.SimpleClaimVerifier

diff --git a/cmd/cosign/cli/verify/verify_attestation.go b/cmd/cosign/cli/verify/verify_attestation.go
@@ -67,6 +67,7 @@ type VerifyAttestationCommand struct {
 	Offline                      bool
 	TSACertChainPath             string
 	IgnoreTlog                   bool
+	MaxWorkers                   int
 }
 
 // Exec runs the verification command
@@ -104,6 +105,7 @@ func (c *VerifyAttestationCommand) Exec(ctx context.Context, images []string) (e
 		Identities:                   identities,
 		Offline:                      c.Offline,
 		IgnoreTlog:                   c.IgnoreTlog,
+		MaxWorkers:                   c.MaxWorkers,
 	}
 	if c.CheckClaims {
 		co.ClaimVerifier = cosign.IntotoSubjectClaimVerifier

diff --git a/doc/cosign_dockerfile_verify.md b/doc/cosign_dockerfile_verify.md
diff --git a/doc/cosign_manifest_verify.md b/doc/cosign_manifest_verify.md
diff --git a/doc/cosign_verify-attestation.md b/doc/cosign_verify-attestation.md
diff --git a/doc/cosign_verify-blob-attestation.md b/doc/cosign_verify-blob-attestation.md
diff --git a/doc/cosign_verify-blob.md b/doc/cosign_verify-blob.md
diff --git a/doc/cosign_verify.md b/doc/cosign_verify.md
diff --git a/go.mod b/go.mod
@@ -26,6 +26,7 @@ require (
 	github.com/mitchellh/go-wordwrap v1.0.1
 	github.com/moby/term v0.5.0
 	github.com/mozillazg/docker-credential-acr-helper v0.3.0
+	github.com/nozzle/throttler v0.0.0-20180817012639-2ea982251481
 	github.com/open-policy-agent/opa v0.52.0
 	github.com/pkg/errors v0.9.1
 	github.com/secure-systems-lab/go-securesystemslib v0.6.0
@@ -52,9 +53,9 @@ require (
 	golang.org/x/sync v0.3.0
 	golang.org/x/term v0.9.0
 	google.golang.org/api v0.128.0
-	k8s.io/api v0.26.6
-	k8s.io/apimachinery v0.26.6
-	k8s.io/client-go v0.26.6
+	k8s.io/api v0.27.3
+	k8s.io/apimachinery v0.27.3
+	k8s.io/client-go v0.27.3
 	k8s.io/utils v0.0.0-20230406110748-d93618cff8a2
 	sigs.k8s.io/release-utils v0.7.4
 )
@@ -84,7 +85,7 @@ require (
 	github.com/AzureAD/microsoft-authentication-library-for-go v1.0.0 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/OneOfOne/xxhash v1.2.8 // indirect
-	github.com/ProtonMail/go-crypto v0.0.0-20230217124315-7d5c6f04bbb8 // indirect
+	github.com/ProtonMail/go-crypto v0.0.0-20230518184743-7afd39499903 // indirect
 	github.com/agnivade/levenshtein v1.1.1 // indirect
 	github.com/alibabacloud-go/alibabacloud-gateway-spi v0.0.4 // indirect
 	github.com/alibabacloud-go/cr-20160607 v1.0.1 // indirect
@@ -145,8 +146,8 @@ require (
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/analysis v0.21.4 // indirect
 	github.com/go-openapi/errors v0.20.3 // indirect
-	github.com/go-openapi/jsonpointer v0.19.5 // indirect
-	github.com/go-openapi/jsonreference v0.20.0 // indirect
+	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/go-openapi/jsonreference v0.20.1 // indirect
 	github.com/go-openapi/loads v0.21.2 // indirect
 	github.com/go-openapi/spec v0.20.9 // indirect
 	github.com/go-openapi/validate v0.22.1 // indirect
@@ -178,7 +179,7 @@ require (
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/vault/api v1.9.2 // indirect
-	github.com/imdario/mergo v0.3.12 // indirect
+	github.com/imdario/mergo v0.3.15 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/jedisct1/go-minisign v0.0.0-20211028175153-1c139d1cc84b // indirect
 	github.com/jellydator/ttlcache/v3 v3.0.1 // indirect
@@ -211,14 +212,14 @@ require (
 	github.com/prometheus/common v0.42.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect
 	github.com/protocolbuffers/txtpbfmt v0.0.0-20220428173112-74888fd59c2b // indirect
-	github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 // indirect
+	github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect
 	github.com/rs/cors v1.9.0 // indirect
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/ryanuber/go-glob v1.0.0 // indirect
 	github.com/sassoftware/relic v7.2.1+incompatible // indirect
 	github.com/segmentio/ksuid v1.0.4 // indirect
 	github.com/shibumi/go-pathspec v1.3.0 // indirect
-	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/sirupsen/logrus v1.9.3 // indirect
 	github.com/skratchdot/open-golang v0.0.0-20200116055534-eef842397966 // indirect
 	github.com/spf13/afero v1.9.5 // indirect
 	github.com/spf13/cast v1.5.1 // indirect
@@ -244,12 +245,12 @@ require (
 	go.uber.org/multierr v1.11.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
 	golang.org/x/exp v0.0.0-20230321023759-10a507213a29 // indirect
-	golang.org/x/mod v0.10.0 // indirect
+	golang.org/x/mod v0.11.0 // indirect
 	golang.org/x/net v0.11.0 // indirect
 	golang.org/x/sys v0.9.0 // indirect
 	golang.org/x/text v0.10.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.8.0 // indirect
+	golang.org/x/tools v0.9.3 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230530153820-e85fd2cbaebc // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect
@@ -262,7 +263,7 @@ require (
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
-	k8s.io/kube-openapi v0.0.0-20221012153701-172d655c2280 // indirect
+	k8s.io/kube-openapi v0.0.0-20230501164219-8b0f38b5fd1f // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect
 	sigs.k8s.io/yaml v1.3.0 // indirect