Refactor bootstrap process to use fluxcd/pkg/git

cmd/flux/bootstrap_bitbucket_server.go

@@ -22,13 +22,13 @@ import (
 	"os"
 	"time"
 
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/fluxcd/pkg/git"
+	"github.com/fluxcd/pkg/git/gogit"
 	"github.com/spf13/cobra"
 
 	"github.com/fluxcd/flux2/internal/flags"
 	"github.com/fluxcd/flux2/internal/utils"
 	"github.com/fluxcd/flux2/pkg/bootstrap"
-	"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
 	"github.com/fluxcd/flux2/pkg/bootstrap/provider"
 	"github.com/fluxcd/flux2/pkg/manifestgen"
 	"github.com/fluxcd/flux2/pkg/manifestgen/install"
@@ -171,10 +171,16 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to create temporary working dir: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
-	gitClient := gogit.New(tmpDir, &http.BasicAuth{
-		Username: user,
-		Password: bitbucketToken,
+
+	gitClient, err := gogit.NewClient(tmpDir, &git.AuthOptions{
+		Transport: git.HTTPS,
+		Username:  user,
+		Password:  bitbucketToken,
+		CAFile:    caBundle,
 	})
+	if err != nil {
+		return err
+	}
 
 	// Install manifest config
 	installOptions := install.Options{
@@ -253,13 +259,12 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error {
 		bootstrap.WithProviderRepository(bServerArgs.owner, bServerArgs.repository, bServerArgs.personal),
 		bootstrap.WithBranch(bootstrapArgs.branch),
 		bootstrap.WithBootstrapTransportType("https"),
-		bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
+		bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
 		bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
 		bootstrap.WithProviderTeamPermissions(mapTeamSlice(bServerArgs.teams, bServerDefaultPermission)),
 		bootstrap.WithReadWriteKeyPermissions(bServerArgs.readWriteKey),
 		bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
 		bootstrap.WithLogger(logger),
-		bootstrap.WithCABundle(caBundle),
 		bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
 	}
 	if bootstrapArgs.sshHostname != "" {

cmd/flux/bootstrap_git.go

@@ -24,21 +24,19 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-git/go-git/v5/plumbing/transport"
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
-	"github.com/go-git/go-git/v5/plumbing/transport/ssh"
 	"github.com/manifoldco/promptui"
 	"github.com/spf13/cobra"
 	corev1 "k8s.io/api/core/v1"
 
 	"github.com/fluxcd/flux2/internal/flags"
 	"github.com/fluxcd/flux2/internal/utils"
 	"github.com/fluxcd/flux2/pkg/bootstrap"
-	"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
 	"github.com/fluxcd/flux2/pkg/manifestgen"
 	"github.com/fluxcd/flux2/pkg/manifestgen/install"
 	"github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret"
 	"github.com/fluxcd/flux2/pkg/manifestgen/sync"
+	"github.com/fluxcd/pkg/git"
+	"github.com/fluxcd/pkg/git/gogit"
 )
 
 var bootstrapGitCmd = &cobra.Command{
@@ -62,6 +60,12 @@ command will perform an upgrade if needed.`,
 
   # Run bootstrap for a Git repository with a private key and password
   flux bootstrap git --url=ssh://git@example.com/repository.git --private-key-file=<path/to/private.key> --password=<password>
+
+  # Run bootstrap for a Git repository on AWS CodeCommit
+  flux bootstrap git --url=ssh://<SSH-Key-ID>@git-codecommit.<region>.amazonaws.com/v1/repos/<repository> --private-key-file=<path/to/private.key> --password=<SSH-passphrase>
+
+  # Run bootstrap for a Git repository on Azure Devops
+  flux bootstrap git --url=ssh://git@ssh.dev.azure.com/v3/<org>/<project>/<repository> --ssh-key-algorithm=rsa --ssh-rsa-bits=4096
 `,
 	RunE: bootstrapGitCmdRun,
 }
@@ -116,9 +120,22 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-	gitAuth, err := transportForURL(repositoryURL)
-	if err != nil {
-		return err
+
+	if strings.Contains(repositoryURL.Hostname(), "git-codecommit") && strings.Contains(repositoryURL.Hostname(), "amazonaws.com") {
+		if repositoryURL.Scheme == string(git.SSH) {
+			if repositoryURL.User == nil {
+				return fmt.Errorf("invalid AWS CodeCommit url: ssh username should be specified in the url")
+			}
+			if repositoryURL.User.Username() == git.DefaultPublicKeyAuthUser {
+				return fmt.Errorf("invalid AWS CodeCommit url: ssh username should be the SSH key ID for the provided private key")
+			}
+			if bootstrapArgs.privateKeyFile == "" {
+				return fmt.Errorf("private key file is required for bootstrapping against AWS CodeCommit using ssh")
+			}
+		}
+		if repositoryURL.Scheme == string(git.HTTPS) && !bootstrapArgs.tokenAuth {
+			return fmt.Errorf("--token-auth=true must be specified for using a HTTPS AWS CodeCommit url")
+		}
 	}
 
 	ctx, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
@@ -145,7 +162,28 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to create temporary working dir: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
-	gitClient := gogit.New(tmpDir, gitAuth)
+
+	var caBundle []byte
+	if bootstrapArgs.caFile != "" {
+		var err error
+		caBundle, err = os.ReadFile(bootstrapArgs.caFile)
+		if err != nil {
+			return fmt.Errorf("unable to read TLS CA file: %w", err)
+		}
+	}
+	authOpts, err := getAuthOpts(repositoryURL, caBundle)
+	if err != nil {
+		return fmt.Errorf("failed to create authentication options for %s: %w", repositoryURL.String(), err)
+	}
+
+	clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
+	if gitArgs.insecureHttpAllowed {
+		clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
+	}
+	gitClient, err := gogit.NewClient(tmpDir, authOpts, clientOpts...)
+	if err != nil {
+		return fmt.Errorf("failed to create a Git client: %w", err)
+	}
 
 	// Install manifest config
 	installOptions := install.Options{
@@ -169,15 +207,6 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
 		installOptions.BaseURL = customBaseURL
 	}
 
-	var caBundle []byte
-	if bootstrapArgs.caFile != "" {
-		var err error
-		caBundle, err = os.ReadFile(bootstrapArgs.caFile)
-		if err != nil {
-			return fmt.Errorf("unable to read TLS CA file: %w", err)
-		}
-	}
-
 	// Source generation and secret config
 	secretOpts := sourcesecret.Options{
 		Name:         bootstrapArgs.secretName,
@@ -253,12 +282,11 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
 	bootstrapOpts := []bootstrap.GitOption{
 		bootstrap.WithRepositoryURL(gitArgs.url),
 		bootstrap.WithBranch(bootstrapArgs.branch),
-		bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
+		bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
 		bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
 		bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
 		bootstrap.WithPostGenerateSecretFunc(promptPublicKey),
 		bootstrap.WithLogger(logger),
-		bootstrap.WithCABundle(caBundle),
 		bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
 	}
 
@@ -272,28 +300,45 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
 	return bootstrap.Run(ctx, b, manifestsBase, installOptions, secretOpts, syncOpts, rootArgs.pollInterval, rootArgs.timeout)
 }
 
-// transportForURL constructs a transport.AuthMethod based on the scheme
+// getAuthOpts retruns a AuthOptions based on the scheme
 // of the given URL and the configured flags. If the protocol equals
 // "ssh" but no private key is configured, authentication using the local
 // SSH-agent is attempted.
-func transportForURL(u *url.URL) (transport.AuthMethod, error) {
+func getAuthOpts(u *url.URL, caBundle []byte) (*git.AuthOptions, error) {
 	switch u.Scheme {
 	case "http":
 		if !gitArgs.insecureHttpAllowed {
 			return nil, fmt.Errorf("scheme http is insecure, pass --allow-insecure-http=true to allow it")
 		}
-		return &http.BasicAuth{
-			Username: gitArgs.username,
-			Password: gitArgs.password,
+		return &git.AuthOptions{
+			Transport: git.HTTP,
+			Username:  gitArgs.username,
+			Password:  gitArgs.password,
 		}, nil
 	case "https":
-		return &http.BasicAuth{
-			Username: gitArgs.username,
-			Password: gitArgs.password,
+		return &git.AuthOptions{
+			Transport: git.HTTPS,
+			Username:  gitArgs.username,
+			Password:  gitArgs.password,
+			CAFile:    caBundle,
 		}, nil
 	case "ssh":
 		if bootstrapArgs.privateKeyFile != "" {
-			return ssh.NewPublicKeysFromFile(u.User.Username(), bootstrapArgs.privateKeyFile, gitArgs.password)
+			pk, err := os.ReadFile(bootstrapArgs.privateKeyFile)
+			if err != nil {
+				return nil, err
+			}
+			kh, err := sourcesecret.ScanHostKey(u.Host)
+			if err != nil {
+				return nil, err
+			}
+			return &git.AuthOptions{
+				Transport:  git.SSH,
+				Username:   u.User.Username(),
+				Password:   gitArgs.password,
+				Identity:   pk,
+				KnownHosts: kh,
+			}, nil
 		}
 		return nil, nil
 	default:

cmd/flux/bootstrap_github.go

@@ -22,13 +22,13 @@ import (
 	"os"
 	"time"
 
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/fluxcd/pkg/git"
+	"github.com/fluxcd/pkg/git/gogit"
 	"github.com/spf13/cobra"
 
 	"github.com/fluxcd/flux2/internal/flags"
 	"github.com/fluxcd/flux2/internal/utils"
 	"github.com/fluxcd/flux2/pkg/bootstrap"
-	"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
 	"github.com/fluxcd/flux2/pkg/bootstrap/provider"
 	"github.com/fluxcd/flux2/pkg/manifestgen"
 	"github.com/fluxcd/flux2/pkg/manifestgen/install"
@@ -161,16 +161,21 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	// Lazy go-git repository
 	tmpDir, err := manifestgen.MkdirTempAbs("", "flux-bootstrap-")
 	if err != nil {
 		return fmt.Errorf("failed to create temporary working dir: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
-	gitClient := gogit.New(tmpDir, &http.BasicAuth{
-		Username: githubArgs.owner,
-		Password: ghToken,
+
+	gitClient, err := gogit.NewClient(tmpDir, &git.AuthOptions{
+		Transport: git.HTTPS,
+		Username:  githubArgs.owner,
+		Password:  ghToken,
+		CAFile:    caBundle,
 	})
+	if err != nil {
+		return err
+	}
 
 	// Install manifest config
 	installOptions := install.Options{
@@ -239,13 +244,12 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
 		bootstrap.WithProviderRepository(githubArgs.owner, githubArgs.repository, githubArgs.personal),
 		bootstrap.WithBranch(bootstrapArgs.branch),
 		bootstrap.WithBootstrapTransportType("https"),
-		bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
+		bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
 		bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
 		bootstrap.WithProviderTeamPermissions(mapTeamSlice(githubArgs.teams, ghDefaultPermission)),
 		bootstrap.WithReadWriteKeyPermissions(githubArgs.readWriteKey),
 		bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
 		bootstrap.WithLogger(logger),
-		bootstrap.WithCABundle(caBundle),
 		bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
 	}
 	if bootstrapArgs.sshHostname != "" {

cmd/flux/bootstrap_gitlab.go

@@ -24,13 +24,13 @@ import (
 	"strings"
 	"time"
 
-	"github.com/go-git/go-git/v5/plumbing/transport/http"
+	"github.com/fluxcd/pkg/git"
+	"github.com/fluxcd/pkg/git/gogit"
 	"github.com/spf13/cobra"
 
 	"github.com/fluxcd/flux2/internal/flags"
 	"github.com/fluxcd/flux2/internal/utils"
 	"github.com/fluxcd/flux2/pkg/bootstrap"
-	"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
 	"github.com/fluxcd/flux2/pkg/bootstrap/provider"
 	"github.com/fluxcd/flux2/pkg/manifestgen"
 	"github.com/fluxcd/flux2/pkg/manifestgen/install"
@@ -178,10 +178,16 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to create temporary working dir: %w", err)
 	}
 	defer os.RemoveAll(tmpDir)
-	gitClient := gogit.New(tmpDir, &http.BasicAuth{
-		Username: gitlabArgs.owner,
-		Password: glToken,
+
+	gitClient, err := gogit.NewClient(tmpDir, &git.AuthOptions{
+		Transport: git.HTTPS,
+		Username:  gitlabArgs.owner,
+		Password:  glToken,
+		CAFile:    caBundle,
 	})
+	if err != nil {
+		return err
+	}
 
 	// Install manifest config
 	installOptions := install.Options{
@@ -255,13 +261,12 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
 		bootstrap.WithProviderRepository(gitlabArgs.owner, gitlabArgs.repository, gitlabArgs.personal),
 		bootstrap.WithBranch(bootstrapArgs.branch),
 		bootstrap.WithBootstrapTransportType("https"),
-		bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
+		bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
 		bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
 		bootstrap.WithProviderTeamPermissions(mapTeamSlice(gitlabArgs.teams, glDefaultPermission)),
 		bootstrap.WithReadWriteKeyPermissions(gitlabArgs.readWriteKey),
 		bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
 		bootstrap.WithLogger(logger),
-		bootstrap.WithCABundle(caBundle),
 		bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
 	}
 	if bootstrapArgs.sshHostname != "" {

go.mod

@@ -8,12 +8,15 @@ require (
 	github.com/cyphar/filepath-securejoin v0.2.3
 	github.com/distribution/distribution/v3 v3.0.0-20221111170714-3b8fbf975279
 	github.com/fluxcd/go-git-providers v0.9.0
+	github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10
 	github.com/fluxcd/helm-controller/api v0.26.0
 	github.com/fluxcd/image-automation-controller/api v0.26.1
 	github.com/fluxcd/image-reflector-controller/api v0.22.1
 	github.com/fluxcd/kustomize-controller/api v0.30.0
 	github.com/fluxcd/notification-controller/api v0.28.0
 	github.com/fluxcd/pkg/apis/meta v0.18.0
+	github.com/fluxcd/pkg/git v0.7.0
+	github.com/fluxcd/pkg/git/gogit v0.2.0
 	github.com/fluxcd/pkg/kustomize v0.10.0
 	github.com/fluxcd/pkg/oci v0.15.0
 	github.com/fluxcd/pkg/runtime v0.24.0
@@ -23,7 +26,6 @@ require (
 	github.com/fluxcd/pkg/untar v0.2.0
 	github.com/fluxcd/pkg/version v0.2.0
 	github.com/fluxcd/source-controller/api v0.31.0
-	github.com/go-git/go-git/v5 v5.4.2
 	github.com/gonvenience/bunt v1.3.4
 	github.com/gonvenience/ytbx v1.4.4
 	github.com/google/go-cmp v0.5.9
@@ -87,7 +89,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.1.2 // indirect
 	github.com/chai2010/gettext-go v1.0.2 // indirect
 	github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
-	github.com/cloudflare/circl v1.1.0 // indirect
+	github.com/cloudflare/circl v1.3.0 // indirect
 	github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
@@ -106,14 +108,14 @@ require (
 	github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
 	github.com/fatih/color v1.13.0 // indirect
 	github.com/felixge/httpsnoop v1.0.1 // indirect
-	github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10 // indirect
 	github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect
 	github.com/fluxcd/pkg/apis/kustomize v0.7.0 // indirect
 	github.com/fluxcd/pkg/tar v0.2.0 // indirect
 	github.com/fsnotify/fsnotify v1.6.0 // indirect
 	github.com/go-errors/errors v1.4.2 // indirect
 	github.com/go-git/gcfg v1.5.0 // indirect
 	github.com/go-git/go-billy/v5 v5.3.1 // indirect
+	github.com/go-git/go-git/v5 v5.4.2 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
@@ -181,6 +183,7 @@ require (
 	github.com/russross/blackfriday/v2 v2.1.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/sirupsen/logrus v1.9.0 // indirect
+	github.com/skeema/knownhosts v1.1.0 // indirect
 	github.com/texttheater/golang-levenshtein v1.0.1 // indirect
 	github.com/vbatts/tar-split v0.11.2 // indirect
 	github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect