Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: http POST #1246

Merged
merged 8 commits into from Nov 18, 2019
Merged

feat: http POST #1246

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d16b99d
feat: http POST
caarlos0 Nov 15, 2019
86c835a
feat: http POST
caarlos0 Nov 15, 2019
c97753e
fix: deprecate
caarlos0 Nov 15, 2019
3fdb740
fix: upload tests
caarlos0 Nov 15, 2019
eebc1cd
Merge remote-tracking branch 'origin/master' into http
caarlos0 Nov 15, 2019
8d6f608
fix: artifactory
caarlos0 Nov 15, 2019
4631837
fix: http username validation
caarlos0 Nov 15, 2019
7ea99df
fix: renames
caarlos0 Nov 17, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
1 change: 1 addition & 0 deletions go.sum
View file
Open in desktop
Expand Up @@ -168,6 +168,7 @@ github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUr
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48=
github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
Expand Down
153 changes: 91 additions & 62 deletions internal/http/http.go
View file
Open in desktop
Expand Up @@ -68,47 +68,74 @@ func assetOpenDefault(kind string, a *artifact.Artifact) (*asset, error) {
}, nil
}

// Defaults sets default configuration options on Put structs
func Defaults(puts []config.Put) error {
for i := range puts {
defaults(&puts[i])
// Defaults sets default configuration options on upload structs
func Defaults(uploads []config.Upload) error {
for i := range uploads {
defaults(&uploads[i])
}
return nil
}

func defaults(put *config.Put) {
if put.Mode == "" {
put.Mode = ModeArchive
func defaults(upload *config.Upload) {
if upload.Mode == "" {
upload.Mode = ModeArchive
}
if upload.Method == "" {
upload.Method = h.MethodPut
}
}

// CheckConfig validates a Put configuration returning a descriptive error when appropriate
func CheckConfig(ctx *context.Context, put *config.Put, kind string) error {
if put.Target == "" {
return misconfigured(kind, put, "missing target")
// CheckConfig validates an upload configuration returning a descriptive error when appropriate
func CheckConfig(ctx *context.Context, upload *config.Upload, kind string) error {
if upload.Target == "" {
return misconfigured(kind, upload, "missing target")
}

if put.Name == "" {
return misconfigured(kind, put, "missing name")
if upload.Name == "" {
return misconfigured(kind, upload, "missing name")
}

if put.Mode != ModeArchive && put.Mode != ModeBinary {
return misconfigured(kind, put, "mode must be 'binary' or 'archive'")
if upload.Mode != ModeArchive && upload.Mode != ModeBinary {
return misconfigured(kind, upload, "mode must be 'binary' or 'archive'")
}

envName := fmt.Sprintf("%s_%s_SECRET", strings.ToUpper(kind), strings.ToUpper(put.Name))
if _, ok := ctx.Env[envName]; !ok {
return misconfigured(kind, put, fmt.Sprintf("missing %s environment variable", envName))
if _, err := getUsername(ctx, upload, kind); err != nil {
return err
}

if _, err := getPassword(ctx, upload, kind); err != nil {
return err
}

if put.TrustedCerts != "" && !x509.NewCertPool().AppendCertsFromPEM([]byte(put.TrustedCerts)) {
return misconfigured(kind, put, "no certificate could be added from the specified trusted_certificates configuration")
if upload.TrustedCerts != "" && !x509.NewCertPool().AppendCertsFromPEM([]byte(upload.TrustedCerts)) {
return misconfigured(kind, upload, "no certificate could be added from the specified trusted_certificates configuration")
}

return nil
}

func misconfigured(kind string, upload *config.Put, reason string) error {
func getUsername(ctx *context.Context, upload *config.Upload, kind string) (string, error) {
if upload.Username != "" {
return upload.Username, nil
}
var key = fmt.Sprintf("%s_%s_USERNAME", strings.ToUpper(kind), strings.ToUpper(upload.Name))
user, ok := ctx.Env[key]
if !ok {
return "", misconfigured(kind, upload, fmt.Sprintf("missing username or %s environment variable", key))
}
return user, nil
}

func getPassword(ctx *context.Context, upload *config.Upload, kind string) (string, error) {
var key = fmt.Sprintf("%s_%s_SECRET", strings.ToUpper(kind), strings.ToUpper(upload.Name))
pwd, ok := ctx.Env[key]
if !ok {
return "", misconfigured(kind, upload, fmt.Sprintf("missing %s environment variable", key))
}
return pwd, nil
}

func misconfigured(kind string, upload *config.Upload, reason string) error {
return pipe.Skip(fmt.Sprintf("%s section '%s' is not configured properly (%s)", kind, upload.Name, reason))
}

Expand All @@ -117,25 +144,25 @@ func misconfigured(kind string, upload *config.Put, reason string) error {
type ResponseChecker func(*h.Response) error

// Upload does the actual uploading work
func Upload(ctx *context.Context, puts []config.Put, kind string, check ResponseChecker) error {
func Upload(ctx *context.Context, uploads []config.Upload, kind string, check ResponseChecker) error {
if ctx.SkipPublish {
return pipe.ErrSkipPublishEnabled
}

// Handle every configured put
for _, put := range puts {
put := put
// Handle every configured upload
for _, upload := range uploads {
upload := upload
filters := []artifact.Filter{}
if put.Checksum {
if upload.Checksum {
filters = append(filters, artifact.ByType(artifact.Checksum))
}
if put.Signature {
if upload.Signature {
filters = append(filters, artifact.ByType(artifact.Signature))
}
// We support two different modes
// - "archive": Upload all artifacts
// - "binary": Upload only the raw binaries
switch v := strings.ToLower(put.Mode); v {
switch v := strings.ToLower(upload.Mode); v {
case ModeArchive:
filters = append(filters,
artifact.ByType(artifact.UploadableArchive),
Expand All @@ -146,52 +173,54 @@ func Upload(ctx *context.Context, puts []config.Put, kind string, check Response
default:
err := fmt.Errorf("%s: mode \"%s\" not supported", kind, v)
log.WithFields(log.Fields{
kind: put.Name,
kind: upload.Name,
"mode": v,
}).Error(err.Error())
return err
}

var filter = artifact.Or(filters...)
if len(put.IDs) > 0 {
filter = artifact.And(filter, artifact.ByIDs(put.IDs...))
if len(upload.IDs) > 0 {
filter = artifact.And(filter, artifact.ByIDs(upload.IDs...))
}
if err := uploadWithFilter(ctx, &put, filter, kind, check); err != nil {
if err := uploadWithFilter(ctx, &upload, filter, kind, check); err != nil {
return err
}
}

return nil
}

func uploadWithFilter(ctx *context.Context, put *config.Put, filter artifact.Filter, kind string, check ResponseChecker) error {
func uploadWithFilter(ctx *context.Context, upload *config.Upload, filter artifact.Filter, kind string, check ResponseChecker) error {
var artifacts = ctx.Artifacts.Filter(filter).List()
log.Debugf("will upload %d artifacts", len(artifacts))
var g = semerrgroup.New(ctx.Parallelism)
for _, artifact := range artifacts {
artifact := artifact
g.Go(func() error {
return uploadAsset(ctx, put, artifact, kind, check)
return uploadAsset(ctx, upload, artifact, kind, check)
})
}
return g.Wait()
}

// uploadAsset uploads file to target and logs all actions
func uploadAsset(ctx *context.Context, put *config.Put, artifact *artifact.Artifact, kind string, check ResponseChecker) error {
envBase := fmt.Sprintf("%s_%s_", strings.ToUpper(kind), strings.ToUpper(put.Name))
username := put.Username
if username == "" {
// username not configured: using env
username = ctx.Env[envBase+"USERNAME"]
func uploadAsset(ctx *context.Context, upload *config.Upload, artifact *artifact.Artifact, kind string, check ResponseChecker) error {
username, err := getUsername(ctx, upload, kind)
if err != nil {
return err
}

secret, err := getPassword(ctx, upload, kind)
if err != nil {
return err
}
secret := ctx.Env[envBase+"SECRET"]

// Generate the target url
targetURL, err := resolveTargetTemplate(ctx, put, artifact)
targetURL, err := resolveTargetTemplate(ctx, upload, artifact)
if err != nil {
msg := fmt.Sprintf("%s: error while building the target url", kind)
log.WithField("instance", put.Name).WithError(err).Error(msg)
log.WithField("instance", upload.Name).WithError(err).Error(msg)
return errors.Wrap(err, msg)
}

Expand All @@ -209,19 +238,19 @@ func uploadAsset(ctx *context.Context, put *config.Put, artifact *artifact.Artif
targetURL += artifact.Name

var headers = map[string]string{}
if put.ChecksumHeader != "" {
if upload.ChecksumHeader != "" {
sum, err := artifact.Checksum("sha256")
if err != nil {
return err
}
headers[put.ChecksumHeader] = sum
headers[upload.ChecksumHeader] = sum
}

res, err := uploadAssetToServer(ctx, put, targetURL, username, secret, headers, asset, check)
res, err := uploadAssetToServer(ctx, upload, targetURL, username, secret, headers, asset, check)
if err != nil {
msg := fmt.Sprintf("%s: upload failed", kind)
log.WithError(err).WithFields(log.Fields{
"instance": put.Name,
"instance": upload.Name,
"username": username,
}).Error(msg)
return errors.Wrap(err, msg)
Expand All @@ -231,26 +260,26 @@ func uploadAsset(ctx *context.Context, put *config.Put, artifact *artifact.Artif
}

log.WithFields(log.Fields{
"instance": put.Name,
"mode": put.Mode,
"instance": upload.Name,
"mode": upload.Mode,
}).Info("uploaded successful")

return nil
}

// uploadAssetToServer uploads the asset file to target
func uploadAssetToServer(ctx *context.Context, put *config.Put, target, username, secret string, headers map[string]string, a *asset, check ResponseChecker) (*h.Response, error) {
req, err := newUploadRequest(target, username, secret, headers, a)
func uploadAssetToServer(ctx *context.Context, upload *config.Upload, target, username, secret string, headers map[string]string, a *asset, check ResponseChecker) (*h.Response, error) {
req, err := newUploadRequest(upload.Method, target, username, secret, headers, a)
if err != nil {
return nil, err
}

return executeHTTPRequest(ctx, put, req, check)
return executeHTTPRequest(ctx, upload, req, check)
}

// newUploadRequest creates a new h.Request for uploading
func newUploadRequest(target, username, secret string, headers map[string]string, a *asset) (*h.Request, error) {
req, err := h.NewRequest(h.MethodPut, target, a.ReadCloser)
func newUploadRequest(method, target, username, secret string, headers map[string]string, a *asset) (*h.Request, error) {
req, err := h.NewRequest(method, target, a.ReadCloser)
if err != nil {
return nil, err
}
Expand All @@ -264,8 +293,8 @@ func newUploadRequest(target, username, secret string, headers map[string]string
return req, err
}

func getHTTPClient(put *config.Put) (*h.Client, error) {
if put.TrustedCerts == "" {
func getHTTPClient(upload *config.Upload) (*h.Client, error) {
if upload.TrustedCerts == "" {
return h.DefaultClient, nil
}
pool, err := x509.SystemCertPool()
Expand All @@ -277,7 +306,7 @@ func getHTTPClient(put *config.Put) (*h.Client, error) {
return nil, err
}
}
pool.AppendCertsFromPEM([]byte(put.TrustedCerts)) // already validated certs checked by CheckConfig
pool.AppendCertsFromPEM([]byte(upload.TrustedCerts)) // already validated certs checked by CheckConfig
return &h.Client{
Transport: &h.Transport{
TLSClientConfig: &tls.Config{
Expand All @@ -288,8 +317,8 @@ func getHTTPClient(put *config.Put) (*h.Client, error) {
}

// executeHTTPRequest processes the http call with respect of context ctx
func executeHTTPRequest(ctx *context.Context, put *config.Put, req *h.Request, check ResponseChecker) (*h.Response, error) {
client, err := getHTTPClient(put)
func executeHTTPRequest(ctx *context.Context, upload *config.Upload, req *h.Request, check ResponseChecker) (*h.Response, error) {
client, err := getHTTPClient(upload)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -334,22 +363,22 @@ type targetData struct {
// resolveTargetTemplate returns the resolved target template with replaced variables
// Those variables can be replaced by the given context, goos, goarch, goarm and more
// TODO: replace this with our internal template pkg
func resolveTargetTemplate(ctx *context.Context, put *config.Put, artifact *artifact.Artifact) (string, error) {
func resolveTargetTemplate(ctx *context.Context, upload *config.Upload, artifact *artifact.Artifact) (string, error) {
data := targetData{
Version: ctx.Version,
Tag: ctx.Git.CurrentTag,
ProjectName: ctx.Config.ProjectName,
}

if put.Mode == ModeBinary {
if upload.Mode == ModeBinary {
// TODO: multiple archives here
data.Os = replace(ctx.Config.Archive.Replacements, artifact.Goos)
data.Arch = replace(ctx.Config.Archive.Replacements, artifact.Goarch)
data.Arm = replace(ctx.Config.Archive.Replacements, artifact.Goarm)
}

var out bytes.Buffer
t, err := template.New(ctx.Config.ProjectName).Parse(put.Target)
t, err := template.New(ctx.Config.ProjectName).Parse(upload.Target)
if err != nil {
return "", err
}
Expand Down