Skip to content

Commit

Permalink
feat: upload source archive (#1379)
Browse files Browse the repository at this point in the history
* feat: upload source archive

Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com>

* fix: lint

Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com>
  • Loading branch information
caarlos0 committed Apr 12, 2020
1 parent 5027d4b commit 7fe4d0a
Show file tree
Hide file tree
Showing 15 changed files with 257 additions and 2 deletions.
15 changes: 13 additions & 2 deletions internal/artifact/artifact.go
Expand Up @@ -47,12 +47,16 @@ const (
Checksum
// Signature is a signature file
Signature
// UploadableSourceArchive is the archive with the current commit source code
UploadableSourceArchive
)

func (t Type) String() string {
switch t {
case UploadableArchive:
return "Archive"
case UploadableFile:
return "File"
case UploadableBinary:
case Binary:
return "Binary"
Expand All @@ -63,10 +67,15 @@ func (t Type) String() string {
case DockerImage:
case PublishableDockerImage:
return "Docker Image"
case PublishableSnapcraft:
case Snapcraft:
return "Snap"
case Checksum:
return "Checksum"
case Signature:
return "Signature"
case UploadableSourceArchive:
return "Source"
}
return "unknown"
}
Expand Down Expand Up @@ -218,8 +227,10 @@ func ByIDs(ids ...string) Filter {
for _, id := range ids {
id := id
filters = append(filters, func(a *Artifact) bool {
// checksum are allways for all artifacts, so return always true.
return a.Type == Checksum || a.ExtraOr("ID", "") == id
// checksum and source archive are always for all artifacts, so return always true.
return a.Type == Checksum ||
a.Type == UploadableSourceArchive ||
a.ExtraOr("ID", "") == id
})
}
return Or(filters...)
Expand Down
1 change: 1 addition & 0 deletions internal/http/http.go
Expand Up @@ -163,6 +163,7 @@ func Upload(ctx *context.Context, uploads []config.Upload, kind string, check Re
// - "binary": Upload only the raw binaries
switch v := strings.ToLower(upload.Mode); v {
case ModeArchive:
// TODO: should we add source archives here too?
filters = append(filters,
artifact.ByType(artifact.UploadableArchive),
artifact.ByType(artifact.LinuxPackage),
Expand Down
42 changes: 42 additions & 0 deletions internal/pipe/blob/blob_minio_test.go
Expand Up @@ -25,8 +25,12 @@ func TestMinioUpload(t *testing.T) {
var listen = randomListen(t)
folder, err := ioutil.TempDir("", "goreleasertest")
assert.NoError(t, err)
srcpath := filepath.Join(folder, "source.tar.gz")
tgzpath := filepath.Join(folder, "bin.tar.gz")
debpath := filepath.Join(folder, "bin.deb")
checkpath := filepath.Join(folder, "check.txt")
assert.NoError(t, ioutil.WriteFile(checkpath, []byte("fake checksums"), 0744))
assert.NoError(t, ioutil.WriteFile(srcpath, []byte("fake\nsrc"), 0744))
assert.NoError(t, ioutil.WriteFile(tgzpath, []byte("fake\ntargz"), 0744))
assert.NoError(t, ioutil.WriteFile(debpath, []byte("fake\ndeb"), 0744))
var ctx = context.New(config.Project{
Expand All @@ -43,6 +47,19 @@ func TestMinioUpload(t *testing.T) {
},
})
ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"}
ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.Checksum,
Name: "checksum.txt",
Path: checkpath,
})
ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.UploadableSourceArchive,
Name: "source.tar.gz",
Path: srcpath,
Extra: map[string]interface{}{
"Format": "tar.gz",
},
})
ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.UploadableArchive,
Name: "bin.tar.gz",
Expand All @@ -65,6 +82,13 @@ func TestMinioUpload(t *testing.T) {
prepareEnv(t, listen)
assert.NoError(t, Pipe{}.Default(ctx))
assert.NoError(t, Pipe{}.Publish(ctx))

require.Subset(t, getFiles(t, ctx, ctx.Config.Blobs[0]), []string{
"testupload/v1.0.0/bin.deb",
"testupload/v1.0.0/bin.tar.gz",
"testupload/v1.0.0/checksum.txt",
"testupload/v1.0.0/source.tar.gz",
})
}

func TestMinioUploadCustomBucketID(t *testing.T) {
Expand Down Expand Up @@ -204,3 +228,21 @@ func stop(t *testing.T, name string) {
func removeTestData(t *testing.T) {
_ = os.RemoveAll("./testdata/data/test/testupload") // dont care if it fails
}

func getFiles(t *testing.T, ctx *context.Context, blob config.Blob) []string {
var bucket = Bucket{}
url, err := bucket.url(ctx, blob)
require.NoError(t, err)
conn, err := bucket.Connect(ctx, url)
require.NoError(t, err)
var iter = conn.List(nil)
var files []string
for {
file, err := iter.Next(ctx)
if err != nil {
break
}
files = append(files, file.Key)
}
return files
}
1 change: 1 addition & 0 deletions internal/pipe/blob/openbucket.go
Expand Up @@ -106,6 +106,7 @@ func (b Bucket) Upload(ctx *context.Context, conf config.Blob) error {
var filter = artifact.Or(
artifact.ByType(artifact.UploadableArchive),
artifact.ByType(artifact.UploadableBinary),
artifact.ByType(artifact.UploadableSourceArchive),
artifact.ByType(artifact.Checksum),
artifact.ByType(artifact.Signature),
artifact.ByType(artifact.LinuxPackage),
Expand Down
1 change: 1 addition & 0 deletions internal/pipe/checksums/checksums.go
Expand Up @@ -39,6 +39,7 @@ func (Pipe) Run(ctx *context.Context) (err error) {
artifact.Or(
artifact.ByType(artifact.UploadableArchive),
artifact.ByType(artifact.UploadableBinary),
artifact.ByType(artifact.UploadableSourceArchive),
artifact.ByType(artifact.LinuxPackage),
),
).List()
Expand Down
2 changes: 2 additions & 0 deletions internal/pipe/checksums/checksums_test.go
Expand Up @@ -163,3 +163,5 @@ func TestDefaultSet(t *testing.T) {
assert.NoError(t, Pipe{}.Default(ctx))
assert.Equal(t, "checksums.txt", ctx.Config.Checksum.NameTemplate)
}

// TODO: add tests for LinuxPackage and UploadableSourceArchive
1 change: 1 addition & 0 deletions internal/pipe/release/release.go
Expand Up @@ -141,6 +141,7 @@ func doPublish(ctx *context.Context, client client.Client) error {
var filters = artifact.Or(
artifact.ByType(artifact.UploadableArchive),
artifact.ByType(artifact.UploadableBinary),
artifact.ByType(artifact.UploadableSourceArchive),
artifact.ByType(artifact.Checksum),
artifact.ByType(artifact.Signature),
artifact.ByType(artifact.LinuxPackage),
Expand Down
11 changes: 11 additions & 0 deletions internal/pipe/release/release_test.go
Expand Up @@ -26,6 +26,8 @@ func TestRunPipeWithoutIDsThenDoesNotFilter(t *testing.T) {
assert.NoError(t, err)
tarfile, err := os.Create(filepath.Join(folder, "bin.tar.gz"))
assert.NoError(t, err)
srcfile, err := os.Create(filepath.Join(folder, "source.tar.gz"))
assert.NoError(t, err)
debfile, err := os.Create(filepath.Join(folder, "bin.deb"))
assert.NoError(t, err)
filteredtarfile, err := os.Create(filepath.Join(folder, "filtered.tar.gz"))
Expand Down Expand Up @@ -76,10 +78,19 @@ func TestRunPipeWithoutIDsThenDoesNotFilter(t *testing.T) {
"ID": "bar",
},
})
ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.UploadableSourceArchive,
Name: "source.tar.gz",
Path: srcfile.Name(),
Extra: map[string]interface{}{
"Format": "tar.gz",
},
})
client := &DummyClient{}
assert.NoError(t, doPublish(ctx, client))
assert.True(t, client.CreatedRelease)
assert.True(t, client.UploadedFile)
assert.Contains(t, client.UploadedFileNames, "source.tar.gz")
assert.Contains(t, client.UploadedFileNames, "bin.deb")
assert.Contains(t, client.UploadedFileNames, "bin.tar.gz")
assert.Contains(t, client.UploadedFileNames, "filtered.deb")
Expand Down
1 change: 1 addition & 0 deletions internal/pipe/sign/sign.go
Expand Up @@ -69,6 +69,7 @@ func (Pipe) Run(ctx *context.Context) error {
filters = append(filters, artifact.Or(
artifact.ByType(artifact.UploadableArchive),
artifact.ByType(artifact.UploadableBinary),
artifact.ByType(artifact.UploadableSourceArchive),
artifact.ByType(artifact.Checksum),
artifact.ByType(artifact.LinuxPackage),
))
Expand Down
59 changes: 59 additions & 0 deletions internal/pipe/sourcearchive/source.go
@@ -0,0 +1,59 @@
// Package sourcearchive archives the source of the project using git-archive.
package sourcearchive

import (
"path/filepath"

"github.com/apex/log"
"github.com/goreleaser/goreleaser/internal/artifact"
"github.com/goreleaser/goreleaser/internal/git"
"github.com/goreleaser/goreleaser/internal/pipe"
"github.com/goreleaser/goreleaser/internal/tmpl"
"github.com/goreleaser/goreleaser/pkg/context"
)

// Pipe for cleandis
type Pipe struct{}

func (Pipe) String() string {
return "creating source archive"
}

// Run the pipe
func (Pipe) Run(ctx *context.Context) (err error) {
if !ctx.Config.Source.Enabled {
return pipe.Skip("source pipe is disabled")
}

name, err := tmpl.New(ctx).Apply(ctx.Config.Source.NameTemplate)
if err != nil {
return err
}
var filename = name + "." + ctx.Config.Source.Format
var path = filepath.Join(ctx.Config.Dist, filename)
log.WithField("file", filename).Info("creating source archive")
out, err := git.Clean(git.Run("archive", "-o", path, ctx.Git.FullCommit))
log.Debug(out)
ctx.Artifacts.Add(&artifact.Artifact{
Type: artifact.UploadableSourceArchive,
Name: filename,
Path: path,
Extra: map[string]interface{}{
"Format": ctx.Config.Source.Format,
},
})
return err
}

// Default sets the pipe defaults
func (Pipe) Default(ctx *context.Context) error {
var archive = &ctx.Config.Source
if archive.Format == "" {
archive.Format = "tar.gz"
}

if archive.NameTemplate == "" {
archive.NameTemplate = "{{ .ProjectName }}-{{ .Version }}"
}
return nil
}
85 changes: 85 additions & 0 deletions internal/pipe/sourcearchive/source_test.go
@@ -0,0 +1,85 @@
package sourcearchive

import (
"io/ioutil"
"os"
"path/filepath"
"testing"

"github.com/goreleaser/goreleaser/internal/artifact"
"github.com/goreleaser/goreleaser/internal/testlib"
"github.com/goreleaser/goreleaser/pkg/config"
"github.com/goreleaser/goreleaser/pkg/context"
"github.com/stretchr/testify/require"
)

func TestArchive(t *testing.T) {
for _, format := range []string{"tar.gz", "tar", "zip"} {
t.Run(format, func(t *testing.T) {
tmp, back := testlib.Mktmp(t)
defer back()
require.NoError(t, os.Mkdir("dist", 0744))

testlib.GitInit(t)
require.NoError(t, ioutil.WriteFile("code.txt", []byte("not really code"), 0655))
require.NoError(t, ioutil.WriteFile("README.md", []byte("# my dope fake project"), 0655))
testlib.GitAdd(t)
testlib.GitCommit(t, "feat: first")

var ctx = context.New(config.Project{
ProjectName: "foo",
Dist: "dist",
Source: config.Source{
Format: format,
Enabled: true,
},
})
ctx.Git.FullCommit = "HEAD"
ctx.Version = "1.0.0"

require.NoError(t, Pipe{}.Default(ctx))
require.NoError(t, Pipe{}.Run(ctx))

var artifacts = ctx.Artifacts.List()
require.Len(t, artifacts, 1)
require.Equal(t, artifact.Artifact{
Type: artifact.UploadableSourceArchive,
Name: "foo-1.0.0." + format,
Path: "dist/foo-1.0.0." + format,
Extra: map[string]interface{}{
"Format": format,
},
}, *artifacts[0])
stat, err := os.Stat(filepath.Join(tmp, "dist", "foo-1.0.0."+format))
require.NoError(t, err)
require.Greater(t, stat.Size(), int64(100))
})
}
}

func TestDefault(t *testing.T) {
var ctx = context.New(config.Project{})
require.NoError(t, Pipe{}.Default(ctx))
require.Equal(t, config.Source{
NameTemplate: "{{ .ProjectName }}-{{ .Version }}",
Format: "tar.gz",
}, ctx.Config.Source)
}

func TestInvalidNameTemplate(t *testing.T) {
var ctx = context.New(config.Project{
Source: config.Source{
Enabled: true,
NameTemplate: "{{ .foo }-asdda",
},
})
require.EqualError(t, Pipe{}.Run(ctx), "template: tmpl:1: unexpected \"}\" in operand")
}

func TestDisabled(t *testing.T) {
testlib.AssertSkipped(t, Pipe{}.Run(context.New(config.Project{})))
}

func TestString(t *testing.T) {
require.NotEmpty(t, Pipe{}.String())
}
2 changes: 2 additions & 0 deletions internal/pipeline/pipeline.go
Expand Up @@ -6,6 +6,7 @@ import (

"github.com/goreleaser/goreleaser/internal/pipe/brew"
"github.com/goreleaser/goreleaser/internal/pipe/semver"
"github.com/goreleaser/goreleaser/internal/pipe/sourcearchive"

"github.com/goreleaser/goreleaser/internal/pipe/archive"
"github.com/goreleaser/goreleaser/internal/pipe/before"
Expand Down Expand Up @@ -48,6 +49,7 @@ var Pipeline = []Piper{
changelog.Pipe{}, // builds the release changelog
build.Pipe{}, // build
archive.Pipe{}, // archive in tar.gz, zip or binary (which does no archiving at all)
sourcearchive.Pipe{}, // archive the source code using git-archive
nfpm.Pipe{}, // archive via fpm (deb, rpm) using "native" go impl
snapcraft.Pipe{}, // archive via snapcraft (snap)
checksums.Pipe{}, // checksums of the files
Expand Down
8 changes: 8 additions & 0 deletions pkg/config/config.go
Expand Up @@ -351,6 +351,13 @@ type Upload struct {
CustomArtifactName bool `yaml:"custom_artifact_name,omitempty"`
}

// Source configuration
type Source struct {
NameTemplate string `yaml:"name_template,omitempty"`
Format string `yaml:",omitempty"`
Enabled bool `yaml:",omitempty"`
}

// Project includes all project configuration
type Project struct {
ProjectName string `yaml:"project_name,omitempty"`
Expand All @@ -374,6 +381,7 @@ type Project struct {
Signs []Sign `yaml:",omitempty"`
EnvFiles EnvFiles `yaml:"env_files,omitempty"`
Before Before `yaml:",omitempty"`
Source Source `yaml:",omitempty"`

// this is a hack ¯\_(ツ)_/¯
SingleBuild Build `yaml:"build,omitempty"`
Expand Down
2 changes: 2 additions & 0 deletions pkg/defaults/defaults.go
Expand Up @@ -19,6 +19,7 @@ import (
"github.com/goreleaser/goreleaser/internal/pipe/sign"
"github.com/goreleaser/goreleaser/internal/pipe/snapcraft"
"github.com/goreleaser/goreleaser/internal/pipe/snapshot"
"github.com/goreleaser/goreleaser/internal/pipe/sourcearchive"
"github.com/goreleaser/goreleaser/pkg/context"
)

Expand All @@ -38,6 +39,7 @@ var Defaulters = []Defaulter{
release.Pipe{},
project.Pipe{},
build.Pipe{},
sourcearchive.Pipe{},
archive.Pipe{},
nfpm.Pipe{},
snapcraft.Pipe{},
Expand Down

0 comments on commit 7fe4d0a

Please sign in to comment.