Skip to content

Commit

Permalink
Merge pull request #3299 from aryan9600/use-pkg-git
Browse files Browse the repository at this point in the history 
Refactor bootstrap process to use `fluxcd/pkg/git`
  • Loading branch information
Paulo Gomes committed Nov 17, 2022
2 parents a06652a + 35e1b5c commit d4ba6c4
Show file tree
Hide file tree
Showing 15 changed files with 262 additions and 596 deletions.
19 changes: 12 additions & 7 deletions cmd/flux/bootstrap_bitbucket_server.go
View file
Expand Up @@ -22,13 +22,13 @@ import (
"os"
"time"

"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/gogit"
"github.com/spf13/cobra"

"github.com/fluxcd/flux2/internal/flags"
"github.com/fluxcd/flux2/internal/utils"
"github.com/fluxcd/flux2/pkg/bootstrap"
"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
"github.com/fluxcd/flux2/pkg/bootstrap/provider"
"github.com/fluxcd/flux2/pkg/manifestgen"
"github.com/fluxcd/flux2/pkg/manifestgen/install"
Expand Down Expand Up @@ -171,10 +171,16 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error {
return fmt.Errorf("failed to create temporary working dir: %w", err)
}
defer os.RemoveAll(tmpDir)
gitClient := gogit.New(tmpDir, &http.BasicAuth{
Username: user,
Password: bitbucketToken,

gitClient, err := gogit.NewClient(tmpDir, &git.AuthOptions{
Transport: git.HTTPS,
Username: user,
Password: bitbucketToken,
CAFile: caBundle,
})
if err != nil {
return err
}

// Install manifest config
installOptions := install.Options{
Expand Down Expand Up @@ -253,13 +259,12 @@ func bootstrapBServerCmdRun(cmd *cobra.Command, args []string) error {
bootstrap.WithProviderRepository(bServerArgs.owner, bServerArgs.repository, bServerArgs.personal),
bootstrap.WithBranch(bootstrapArgs.branch),
bootstrap.WithBootstrapTransportType("https"),
bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
bootstrap.WithProviderTeamPermissions(mapTeamSlice(bServerArgs.teams, bServerDefaultPermission)),
bootstrap.WithReadWriteKeyPermissions(bServerArgs.readWriteKey),
bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
bootstrap.WithLogger(logger),
bootstrap.WithCABundle(caBundle),
bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
}
if bootstrapArgs.sshHostname != "" {
Expand Down
101 changes: 73 additions & 28 deletions cmd/flux/bootstrap_git.go
View file
Expand Up @@ -24,21 +24,19 @@ import (
"strings"
"time"

"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/manifoldco/promptui"
"github.com/spf13/cobra"
corev1 "k8s.io/api/core/v1"

"github.com/fluxcd/flux2/internal/flags"
"github.com/fluxcd/flux2/internal/utils"
"github.com/fluxcd/flux2/pkg/bootstrap"
"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
"github.com/fluxcd/flux2/pkg/manifestgen"
"github.com/fluxcd/flux2/pkg/manifestgen/install"
"github.com/fluxcd/flux2/pkg/manifestgen/sourcesecret"
"github.com/fluxcd/flux2/pkg/manifestgen/sync"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/gogit"
)

var bootstrapGitCmd = &cobra.Command{
Expand All @@ -62,6 +60,12 @@ command will perform an upgrade if needed.`,
# Run bootstrap for a Git repository with a private key and password
flux bootstrap git --url=ssh://git@example.com/repository.git --private-key-file=<path/to/private.key> --password=<password>
# Run bootstrap for a Git repository on AWS CodeCommit
flux bootstrap git --url=ssh://<SSH-Key-ID>@git-codecommit.<region>.amazonaws.com/v1/repos/<repository> --private-key-file=<path/to/private.key> --password=<SSH-passphrase>
# Run bootstrap for a Git repository on Azure Devops
flux bootstrap git --url=ssh://git@ssh.dev.azure.com/v3/<org>/<project>/<repository> --ssh-key-algorithm=rsa --ssh-rsa-bits=4096
`,
RunE: bootstrapGitCmdRun,
}
Expand Down Expand Up @@ -116,9 +120,22 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
gitAuth, err := transportForURL(repositoryURL)
if err != nil {
return err

if strings.Contains(repositoryURL.Hostname(), "git-codecommit") && strings.Contains(repositoryURL.Hostname(), "amazonaws.com") {
if repositoryURL.Scheme == string(git.SSH) {
if repositoryURL.User == nil {
return fmt.Errorf("invalid AWS CodeCommit url: ssh username should be specified in the url")
}
if repositoryURL.User.Username() == git.DefaultPublicKeyAuthUser {
return fmt.Errorf("invalid AWS CodeCommit url: ssh username should be the SSH key ID for the provided private key")
}
if bootstrapArgs.privateKeyFile == "" {
return fmt.Errorf("private key file is required for bootstrapping against AWS CodeCommit using ssh")
}
}
if repositoryURL.Scheme == string(git.HTTPS) && !bootstrapArgs.tokenAuth {
return fmt.Errorf("--token-auth=true must be specified for using a HTTPS AWS CodeCommit url")
}
}

ctx, cancel := context.WithTimeout(context.Background(), rootArgs.timeout)
Expand All @@ -145,7 +162,28 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
return fmt.Errorf("failed to create temporary working dir: %w", err)
}
defer os.RemoveAll(tmpDir)
gitClient := gogit.New(tmpDir, gitAuth)

var caBundle []byte
if bootstrapArgs.caFile != "" {
var err error
caBundle, err = os.ReadFile(bootstrapArgs.caFile)
if err != nil {
return fmt.Errorf("unable to read TLS CA file: %w", err)
}
}
authOpts, err := getAuthOpts(repositoryURL, caBundle)
if err != nil {
return fmt.Errorf("failed to create authentication options for %s: %w", repositoryURL.String(), err)
}

clientOpts := []gogit.ClientOption{gogit.WithDiskStorage()}
if gitArgs.insecureHttpAllowed {
clientOpts = append(clientOpts, gogit.WithInsecureCredentialsOverHTTP())
}
gitClient, err := gogit.NewClient(tmpDir, authOpts, clientOpts...)
if err != nil {
return fmt.Errorf("failed to create a Git client: %w", err)
}

// Install manifest config
installOptions := install.Options{
Expand All @@ -169,15 +207,6 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
installOptions.BaseURL = customBaseURL
}

var caBundle []byte
if bootstrapArgs.caFile != "" {
var err error
caBundle, err = os.ReadFile(bootstrapArgs.caFile)
if err != nil {
return fmt.Errorf("unable to read TLS CA file: %w", err)
}
}

// Source generation and secret config
secretOpts := sourcesecret.Options{
Name: bootstrapArgs.secretName,
Expand Down Expand Up @@ -253,12 +282,11 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
bootstrapOpts := []bootstrap.GitOption{
bootstrap.WithRepositoryURL(gitArgs.url),
bootstrap.WithBranch(bootstrapArgs.branch),
bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
bootstrap.WithPostGenerateSecretFunc(promptPublicKey),
bootstrap.WithLogger(logger),
bootstrap.WithCABundle(caBundle),
bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
}

Expand All @@ -272,28 +300,45 @@ func bootstrapGitCmdRun(cmd *cobra.Command, args []string) error {
return bootstrap.Run(ctx, b, manifestsBase, installOptions, secretOpts, syncOpts, rootArgs.pollInterval, rootArgs.timeout)
}

// transportForURL constructs a transport.AuthMethod based on the scheme
// getAuthOpts retruns a AuthOptions based on the scheme
// of the given URL and the configured flags. If the protocol equals
// "ssh" but no private key is configured, authentication using the local
// SSH-agent is attempted.
func transportForURL(u *url.URL) (transport.AuthMethod, error) {
func getAuthOpts(u *url.URL, caBundle []byte) (*git.AuthOptions, error) {
switch u.Scheme {
case "http":
if !gitArgs.insecureHttpAllowed {
return nil, fmt.Errorf("scheme http is insecure, pass --allow-insecure-http=true to allow it")
}
return &http.BasicAuth{
Username: gitArgs.username,
Password: gitArgs.password,
return &git.AuthOptions{
Transport: git.HTTP,
Username: gitArgs.username,
Password: gitArgs.password,
}, nil
case "https":
return &http.BasicAuth{
Username: gitArgs.username,
Password: gitArgs.password,
return &git.AuthOptions{
Transport: git.HTTPS,
Username: gitArgs.username,
Password: gitArgs.password,
CAFile: caBundle,
}, nil
case "ssh":
if bootstrapArgs.privateKeyFile != "" {
return ssh.NewPublicKeysFromFile(u.User.Username(), bootstrapArgs.privateKeyFile, gitArgs.password)
pk, err := os.ReadFile(bootstrapArgs.privateKeyFile)
if err != nil {
return nil, err
}
kh, err := sourcesecret.ScanHostKey(u.Host)
if err != nil {
return nil, err
}
return &git.AuthOptions{
Transport: git.SSH,
Username: u.User.Username(),
Password: gitArgs.password,
Identity: pk,
KnownHosts: kh,
}, nil
}
return nil, nil
default:
Expand Down
20 changes: 12 additions & 8 deletions cmd/flux/bootstrap_github.go
View file
Expand Up @@ -22,13 +22,13 @@ import (
"os"
"time"

"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/gogit"
"github.com/spf13/cobra"

"github.com/fluxcd/flux2/internal/flags"
"github.com/fluxcd/flux2/internal/utils"
"github.com/fluxcd/flux2/pkg/bootstrap"
"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
"github.com/fluxcd/flux2/pkg/bootstrap/provider"
"github.com/fluxcd/flux2/pkg/manifestgen"
"github.com/fluxcd/flux2/pkg/manifestgen/install"
Expand Down Expand Up @@ -161,16 +161,21 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
return err
}

// Lazy go-git repository
tmpDir, err := manifestgen.MkdirTempAbs("", "flux-bootstrap-")
if err != nil {
return fmt.Errorf("failed to create temporary working dir: %w", err)
}
defer os.RemoveAll(tmpDir)
gitClient := gogit.New(tmpDir, &http.BasicAuth{
Username: githubArgs.owner,
Password: ghToken,

gitClient, err := gogit.NewClient(tmpDir, &git.AuthOptions{
Transport: git.HTTPS,
Username: githubArgs.owner,
Password: ghToken,
CAFile: caBundle,
})
if err != nil {
return err
}

// Install manifest config
installOptions := install.Options{
Expand Down Expand Up @@ -239,13 +244,12 @@ func bootstrapGitHubCmdRun(cmd *cobra.Command, args []string) error {
bootstrap.WithProviderRepository(githubArgs.owner, githubArgs.repository, githubArgs.personal),
bootstrap.WithBranch(bootstrapArgs.branch),
bootstrap.WithBootstrapTransportType("https"),
bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
bootstrap.WithProviderTeamPermissions(mapTeamSlice(githubArgs.teams, ghDefaultPermission)),
bootstrap.WithReadWriteKeyPermissions(githubArgs.readWriteKey),
bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
bootstrap.WithLogger(logger),
bootstrap.WithCABundle(caBundle),
bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
}
if bootstrapArgs.sshHostname != "" {
Expand Down
19 changes: 12 additions & 7 deletions cmd/flux/bootstrap_gitlab.go
View file
Expand Up @@ -24,13 +24,13 @@ import (
"strings"
"time"

"github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/fluxcd/pkg/git"
"github.com/fluxcd/pkg/git/gogit"
"github.com/spf13/cobra"

"github.com/fluxcd/flux2/internal/flags"
"github.com/fluxcd/flux2/internal/utils"
"github.com/fluxcd/flux2/pkg/bootstrap"
"github.com/fluxcd/flux2/pkg/bootstrap/git/gogit"
"github.com/fluxcd/flux2/pkg/bootstrap/provider"
"github.com/fluxcd/flux2/pkg/manifestgen"
"github.com/fluxcd/flux2/pkg/manifestgen/install"
Expand Down Expand Up @@ -178,10 +178,16 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
return fmt.Errorf("failed to create temporary working dir: %w", err)
}
defer os.RemoveAll(tmpDir)
gitClient := gogit.New(tmpDir, &http.BasicAuth{
Username: gitlabArgs.owner,
Password: glToken,

gitClient, err := gogit.NewClient(tmpDir, &git.AuthOptions{
Transport: git.HTTPS,
Username: gitlabArgs.owner,
Password: glToken,
CAFile: caBundle,
})
if err != nil {
return err
}

// Install manifest config
installOptions := install.Options{
Expand Down Expand Up @@ -255,13 +261,12 @@ func bootstrapGitLabCmdRun(cmd *cobra.Command, args []string) error {
bootstrap.WithProviderRepository(gitlabArgs.owner, gitlabArgs.repository, gitlabArgs.personal),
bootstrap.WithBranch(bootstrapArgs.branch),
bootstrap.WithBootstrapTransportType("https"),
bootstrap.WithAuthor(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithSignature(bootstrapArgs.authorName, bootstrapArgs.authorEmail),
bootstrap.WithCommitMessageAppendix(bootstrapArgs.commitMessageAppendix),
bootstrap.WithProviderTeamPermissions(mapTeamSlice(gitlabArgs.teams, glDefaultPermission)),
bootstrap.WithReadWriteKeyPermissions(gitlabArgs.readWriteKey),
bootstrap.WithKubeconfig(kubeconfigArgs, kubeclientOptions),
bootstrap.WithLogger(logger),
bootstrap.WithCABundle(caBundle),
bootstrap.WithGitCommitSigning(entityList, bootstrapArgs.gpgPassphrase, bootstrapArgs.gpgKeyID),
}
if bootstrapArgs.sshHostname != "" {
Expand Down
9 changes: 6 additions & 3 deletions go.mod
View file
Expand Up @@ -8,12 +8,15 @@ require (
github.com/cyphar/filepath-securejoin v0.2.3
github.com/distribution/distribution/v3 v3.0.0-20221111170714-3b8fbf975279
github.com/fluxcd/go-git-providers v0.9.0
github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10
github.com/fluxcd/helm-controller/api v0.26.0
github.com/fluxcd/image-automation-controller/api v0.26.1
github.com/fluxcd/image-reflector-controller/api v0.22.1
github.com/fluxcd/kustomize-controller/api v0.30.0
github.com/fluxcd/notification-controller/api v0.28.0
github.com/fluxcd/pkg/apis/meta v0.18.0
github.com/fluxcd/pkg/git v0.7.0
github.com/fluxcd/pkg/git/gogit v0.2.0
github.com/fluxcd/pkg/kustomize v0.10.0
github.com/fluxcd/pkg/oci v0.15.0
github.com/fluxcd/pkg/runtime v0.24.0
Expand All @@ -23,7 +26,6 @@ require (
github.com/fluxcd/pkg/untar v0.2.0
github.com/fluxcd/pkg/version v0.2.0
github.com/fluxcd/source-controller/api v0.31.0
github.com/go-git/go-git/v5 v5.4.2
github.com/gonvenience/bunt v1.3.4
github.com/gonvenience/ytbx v1.4.4
github.com/google/go-cmp v0.5.9
Expand Down Expand Up @@ -87,7 +89,7 @@ require (
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/chai2010/gettext-go v1.0.2 // indirect
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect
github.com/cloudflare/circl v1.1.0 // indirect
github.com/cloudflare/circl v1.3.0 // indirect
github.com/containerd/stargz-snapshotter/estargz v0.12.1 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
Expand All @@ -106,14 +108,14 @@ require (
github.com/exponent-io/jsonpath v0.0.0-20210407135951-1de76d718b3f // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/felixge/httpsnoop v1.0.1 // indirect
github.com/fluxcd/go-git/v5 v5.0.0-20221104190732-329fd6659b10 // indirect
github.com/fluxcd/pkg/apis/acl v0.1.0 // indirect
github.com/fluxcd/pkg/apis/kustomize v0.7.0 // indirect
github.com/fluxcd/pkg/tar v0.2.0 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-errors/errors v1.4.2 // indirect
github.com/go-git/gcfg v1.5.0 // indirect
github.com/go-git/go-billy/v5 v5.3.1 // indirect
github.com/go-git/go-git/v5 v5.4.2 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-openapi/jsonpointer v0.19.5 // indirect
github.com/go-openapi/jsonreference v0.20.0 // indirect
Expand Down Expand Up @@ -181,6 +183,7 @@ require (
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.2.0 // indirect
github.com/sirupsen/logrus v1.9.0 // indirect
github.com/skeema/knownhosts v1.1.0 // indirect
github.com/texttheater/golang-levenshtein v1.0.1 // indirect
github.com/vbatts/tar-split v0.11.2 // indirect
github.com/virtuald/go-ordered-json v0.0.0-20170621173500-b18e6e673d74 // indirect
Expand Down

0 comments on commit d4ba6c4

Please sign in to comment.