diff --git a/internal/artifact/artifact.go b/internal/artifact/artifact.go index e38c40d811f..fb161ad2e9e 100644 --- a/internal/artifact/artifact.go +++ b/internal/artifact/artifact.go @@ -27,6 +27,8 @@ const ( UploadableArchive Type = iota // UploadableBinary is a binary file to be uploaded UploadableBinary + // UploadableFile is any file that can be uploaded + UploadableFile // Binary is a binary (output of a gobuild) Binary // LinuxPackage is a linux package generated by nfpm diff --git a/internal/pipe/release/release.go b/internal/pipe/release/release.go index 4b31fd122af..d8783dc0f8a 100644 --- a/internal/pipe/release/release.go +++ b/internal/pipe/release/release.go @@ -2,6 +2,7 @@ package release import ( "os" + "path/filepath" "time" "github.com/apex/log" @@ -13,6 +14,7 @@ import ( "github.com/kamilsk/retry/v4" "github.com/kamilsk/retry/v4/backoff" "github.com/kamilsk/retry/v4/strategy" + "github.com/mattn/go-zglob" "github.com/pkg/errors" ) @@ -123,55 +125,96 @@ func doPublish(ctx *context.Context, client client.Client) error { return err } - var filters = []artifact.Filter{ - artifact.Or( - artifact.ByType(artifact.UploadableArchive), - artifact.ByType(artifact.UploadableBinary), - artifact.ByType(artifact.Checksum), - artifact.ByType(artifact.Signature), - artifact.ByType(artifact.LinuxPackage), - ), + extraFiles, err := findFiles(ctx) + if err != nil { + return err + } + + for name, path := range extraFiles { + if _, err := os.Stat(path); os.IsNotExist(err) { + return errors.Wrapf(err, "failed to upload %s", name) + } + ctx.Artifacts.Add(&artifact.Artifact{ + Name: name, + Path: path, + Type: artifact.UploadableFile, + }) } + var filters = artifact.Or( + artifact.ByType(artifact.UploadableArchive), + artifact.ByType(artifact.UploadableBinary), + artifact.ByType(artifact.Checksum), + artifact.ByType(artifact.Signature), + artifact.ByType(artifact.LinuxPackage), + ) + if len(ctx.Config.Release.IDs) > 0 { - filters = append(filters, artifact.ByIDs(ctx.Config.Release.IDs...)) + filters = artifact.And(filters, artifact.ByIDs(ctx.Config.Release.IDs...)) } + filters = artifact.Or(filters, artifact.ByType(artifact.UploadableFile)) + var g = semerrgroup.New(ctx.Parallelism) - for _, artifact := range ctx.Artifacts.Filter(artifact.And(filters...)).List() { + for _, artifact := range ctx.Artifacts.Filter(filters).List() { artifact := artifact g.Go(func() error { - var repeats uint - what := func(try uint) error { - repeats = try + 1 - if uploadErr := upload(ctx, client, releaseID, artifact); uploadErr != nil { - log.WithFields(log.Fields{ - "try": try, - "artifact": artifact.Name, - }).Warnf("failed to upload artifact, will retry") - return uploadErr - } - return nil - } - how := []func(uint, error) bool{ - strategy.Limit(10), - strategy.Backoff(backoff.Linear(50 * time.Millisecond)), - } - if err := retry.Try(ctx, what, how...); err != nil { - return errors.Wrapf(err, "failed to upload %s after %d retries", artifact.Name, repeats) - } - return nil + return upload(ctx, client, releaseID, artifact) }) } return g.Wait() } func upload(ctx *context.Context, client client.Client, releaseID string, artifact *artifact.Artifact) error { - file, err := os.Open(artifact.Path) - if err != nil { - return err + var repeats uint + what := func(try uint) error { + repeats = try + 1 + file, err := os.Open(artifact.Path) + if err != nil { + return err + } + defer file.Close() // nolint: errcheck + log.WithField("file", file.Name()).WithField("name", artifact.Name).Info("uploading to release") + if err := client.Upload(ctx, releaseID, artifact, file); err != nil { + log.WithFields(log.Fields{ + "try": try, + "artifact": artifact.Name, + }).Warnf("failed to upload artifact, will retry") + return err + } + return nil + } + how := []func(uint, error) bool{ + strategy.Limit(10), + strategy.Backoff(backoff.Linear(50 * time.Millisecond)), + } + if err := retry.Try(ctx, what, how...); err != nil { + return errors.Wrapf(err, "failed to upload %s after %d retries", artifact.Name, repeats) + } + return nil +} + +func findFiles(ctx *context.Context) (map[string]string, error) { + var result = map[string]string{} + for _, extra := range ctx.Config.Release.ExtraFiles { + if extra.Glob != "" { + files, err := zglob.Glob(extra.Glob) + if err != nil { + return result, errors.Wrapf(err, "globbing failed for pattern %s", extra.Glob) + } + for _, file := range files { + info, err := os.Stat(file) + if err == nil && info.IsDir() { + log.Debugf("ignoring directory %s", file) + continue + } + var name = filepath.Base(file) + if old, ok := result[name]; ok { + log.Warnf("overriding %s with %s for name %s", old, file, name) + } + result[name] = file + } + } } - defer file.Close() // nolint: errcheck - log.WithField("file", file.Name()).WithField("name", artifact.Name).Info("uploading to release") - return client.Upload(ctx, releaseID, artifact, file) + return result, nil } diff --git a/internal/pipe/release/release_test.go b/internal/pipe/release/release_test.go index 898883955d1..1334c66c745 100644 --- a/internal/pipe/release/release_test.go +++ b/internal/pipe/release/release_test.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strings" "sync" "testing" @@ -104,6 +105,9 @@ func TestRunPipeWithIDsThenFilters(t *testing.T) { Name: "test", }, IDs: []string{"foo"}, + ExtraFiles: []config.ExtraFile{ + {Glob: "./testdata/**/*"}, + }, }, } var ctx = context.New(config) @@ -146,6 +150,9 @@ func TestRunPipeWithIDsThenFilters(t *testing.T) { assert.True(t, client.UploadedFile) assert.Contains(t, client.UploadedFileNames, "bin.deb") assert.Contains(t, client.UploadedFileNames, "bin.tar.gz") + assert.Contains(t, client.UploadedFileNames, "release1.golden") + assert.Contains(t, client.UploadedFileNames, "release2.golden") + assert.Contains(t, client.UploadedFileNames, "f1") assert.NotContains(t, client.UploadedFileNames, "filtered.deb") assert.NotContains(t, client.UploadedFileNames, "filtered.tar.gz") } @@ -219,6 +226,50 @@ func TestRunPipeUploadFailure(t *testing.T) { assert.False(t, client.UploadedFile) } +func TestRunPipeExtraFileNotFound(t *testing.T) { + var config = config.Project{ + Release: config.Release{ + GitHub: config.Repo{ + Owner: "test", + Name: "test", + }, + ExtraFiles: []config.ExtraFile{ + {Glob: "./testdata/release2.golden"}, + {Glob: "./nope"}, + }, + }, + } + var ctx = context.New(config) + ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"} + client := &DummyClient{} + assert.EqualError(t, doPublish(ctx, client), "globbing failed for pattern ./nope: file does not exist") + assert.True(t, client.CreatedRelease) + assert.False(t, client.UploadedFile) +} + +func TestRunPipeExtraOverride(t *testing.T) { + var config = config.Project{ + Release: config.Release{ + GitHub: config.Repo{ + Owner: "test", + Name: "test", + }, + ExtraFiles: []config.ExtraFile{ + {Glob: "./testdata/**/*"}, + {Glob: "./testdata/upload_same_name/f1"}, + }, + }, + } + var ctx = context.New(config) + ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"} + client := &DummyClient{} + assert.NoError(t, doPublish(ctx, client)) + assert.True(t, client.CreatedRelease) + assert.True(t, client.UploadedFile) + assert.Contains(t, client.UploadedFileNames, "f1") + assert.True(t, strings.HasSuffix(client.UploadedFilePaths["f1"], "testdata/upload_same_name/f1")) +} + func TestRunPipeUploadRetry(t *testing.T) { folder, err := ioutil.TempDir("", "goreleasertest") assert.NoError(t, err) @@ -470,6 +521,7 @@ type DummyClient struct { CreatedRelease bool UploadedFile bool UploadedFileNames []string + UploadedFilePaths map[string]string FailFirstUpload bool Lock sync.Mutex } @@ -489,6 +541,9 @@ func (client *DummyClient) CreateFile(ctx *context.Context, commitAuthor config. func (client *DummyClient) Upload(ctx *context.Context, releaseID string, artifact *artifact.Artifact, file *os.File) error { client.Lock.Lock() defer client.Lock.Unlock() + if client.UploadedFilePaths == nil { + client.UploadedFilePaths = map[string]string{} + } // ensure file is read to better mimic real behavior _, err := ioutil.ReadAll(file) if err != nil { @@ -503,5 +558,6 @@ func (client *DummyClient) Upload(ctx *context.Context, releaseID string, artifa } client.UploadedFile = true client.UploadedFileNames = append(client.UploadedFileNames, artifact.Name) + client.UploadedFilePaths[artifact.Name] = artifact.Path return nil } diff --git a/internal/pipe/release/testdata/upload_same_name/f1 b/internal/pipe/release/testdata/upload_same_name/f1 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/internal/pipe/release/testdata/upload_same_name/f2/f1 b/internal/pipe/release/testdata/upload_same_name/f2/f1 new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pkg/config/config.go b/pkg/config/config.go index ac492f21769..7e6ae8ada21 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -178,14 +178,20 @@ type Archive struct { // Release config used for the GitHub/GitLab release type Release struct { - GitHub Repo `yaml:",omitempty"` - GitLab Repo `yaml:",omitempty"` - Gitea Repo `yaml:",omitempty"` - Draft bool `yaml:",omitempty"` - Disable bool `yaml:",omitempty"` - Prerelease string `yaml:",omitempty"` - NameTemplate string `yaml:"name_template,omitempty"` - IDs []string `yaml:"ids,omitempty"` + GitHub Repo `yaml:",omitempty"` + GitLab Repo `yaml:",omitempty"` + Gitea Repo `yaml:",omitempty"` + Draft bool `yaml:",omitempty"` + Disable bool `yaml:",omitempty"` + Prerelease string `yaml:",omitempty"` + NameTemplate string `yaml:"name_template,omitempty"` + IDs []string `yaml:"ids,omitempty"` + ExtraFiles []ExtraFile `yaml:"extra_files,omitempty"` +} + +// ExtraFile on a release +type ExtraFile struct { + Glob string `yaml:"glob,omitempty"` } // NFPM config diff --git a/www/content/release.md b/www/content/release.md index fa585bdea45..3552a7540c7 100644 --- a/www/content/release.md +++ b/www/content/release.md @@ -45,6 +45,15 @@ release: # GitHub. # Defaults to false. disable: true + + # You can add extra pre-existing files to the release. + # The filename on the release will be the last part of the path (base). If + # another file with the same name exists, the latest one found will be used. + # Defaults to empty. + extra_files: + - glob: ./path/to/file.txt + - glob: ./glob/**/to/**/file/**/* + - glob: ./glob/foo/to/bar/file/foobar/override_from_previous ``` Second, let's see what can be customized in the `release` section for GitLab. @@ -73,6 +82,15 @@ release: # GitLab. # Defaults to false. disable: true + + # You can add extra pre-existing files to the release. + # The filename on the release will be the last part of the path (base). If + # another file with the same name exists, the latest one found will be used. + # Defaults to empty. + extra_files: + - glob: ./path/to/file.txt + - glob: ./glob/**/to/**/file/**/* + - glob: ./glob/foo/to/bar/file/foobar/override_from_previous ``` You can also configure the `release` section to upload to a [Gitea](https://gitea.io) instance: @@ -99,6 +117,15 @@ release: # Gitea. # Defaults to false. disable: true + + # You can add extra pre-existing files to the release. + # The filename on the release will be the last part of the path (base). If + # another file with the same name exists, the latest one found will be used. + # Defaults to empty. + extra_files: + - glob: ./path/to/file.txt + - glob: ./glob/**/to/**/file/**/* + - glob: ./glob/foo/to/bar/file/foobar/override_from_previous ``` To enable uploading `tar.gz` and `checksums.txt` files you need to add the following to your Gitea config in `app.ini`: