feat: upload extra files to the release (#1333)

* feat: upload extra files to the release Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com> * fix: retry upload Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com> * fix: go mod tidy Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com> * fix: globs Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com> * fix: globs Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com> * fix: typo Signed-off-by: Carlos Alexandro Becker <caarlos0@gmail.com>
goreleaser · Feb 11, 2020 · d7c5405 · d7c5405
1 parent dd35f28
commit d7c5405
Show file tree

Hide file tree

Showing 7 changed files with 178 additions and 44 deletions.
diff --git a/internal/artifact/artifact.go b/internal/artifact/artifact.go
@@ -27,6 +27,8 @@ const (
 	UploadableArchive Type = iota
 	// UploadableBinary is a binary file to be uploaded
 	UploadableBinary
+	// UploadableFile is any file that can be uploaded
+	UploadableFile
 	// Binary is a binary (output of a gobuild)
 	Binary
 	// LinuxPackage is a linux package generated by nfpm

diff --git a/internal/pipe/release/release.go b/internal/pipe/release/release.go
@@ -2,6 +2,7 @@ package release
 
 import (
 	"os"
+	"path/filepath"
 	"time"
 
 	"github.com/apex/log"
@@ -13,6 +14,7 @@ import (
 	"github.com/kamilsk/retry/v4"
 	"github.com/kamilsk/retry/v4/backoff"
 	"github.com/kamilsk/retry/v4/strategy"
+	"github.com/mattn/go-zglob"
 	"github.com/pkg/errors"
 )
 
@@ -123,55 +125,96 @@ func doPublish(ctx *context.Context, client client.Client) error {
 		return err
 	}
 
-	var filters = []artifact.Filter{
-		artifact.Or(
-			artifact.ByType(artifact.UploadableArchive),
-			artifact.ByType(artifact.UploadableBinary),
-			artifact.ByType(artifact.Checksum),
-			artifact.ByType(artifact.Signature),
-			artifact.ByType(artifact.LinuxPackage),
-		),
+	extraFiles, err := findFiles(ctx)
+	if err != nil {
+		return err
+	}
+
+	for name, path := range extraFiles {
+		if _, err := os.Stat(path); os.IsNotExist(err) {
+			return errors.Wrapf(err, "failed to upload %s", name)
+		}
+		ctx.Artifacts.Add(&artifact.Artifact{
+			Name: name,
+			Path: path,
+			Type: artifact.UploadableFile,
+		})
 	}
 
+	var filters = artifact.Or(
+		artifact.ByType(artifact.UploadableArchive),
+		artifact.ByType(artifact.UploadableBinary),
+		artifact.ByType(artifact.Checksum),
+		artifact.ByType(artifact.Signature),
+		artifact.ByType(artifact.LinuxPackage),
+	)
+
 	if len(ctx.Config.Release.IDs) > 0 {
-		filters = append(filters, artifact.ByIDs(ctx.Config.Release.IDs...))
+		filters = artifact.And(filters, artifact.ByIDs(ctx.Config.Release.IDs...))
 	}
 
+	filters = artifact.Or(filters, artifact.ByType(artifact.UploadableFile))
+
 	var g = semerrgroup.New(ctx.Parallelism)
-	for _, artifact := range ctx.Artifacts.Filter(artifact.And(filters...)).List() {
+	for _, artifact := range ctx.Artifacts.Filter(filters).List() {
 		artifact := artifact
 		g.Go(func() error {
-			var repeats uint
-			what := func(try uint) error {
-				repeats = try + 1
-				if uploadErr := upload(ctx, client, releaseID, artifact); uploadErr != nil {
-					log.WithFields(log.Fields{
-						"try":      try,
-						"artifact": artifact.Name,
-					}).Warnf("failed to upload artifact, will retry")
-					return uploadErr
-				}
-				return nil
-			}
-			how := []func(uint, error) bool{
-				strategy.Limit(10),
-				strategy.Backoff(backoff.Linear(50 * time.Millisecond)),
-			}
-			if err := retry.Try(ctx, what, how...); err != nil {
-				return errors.Wrapf(err, "failed to upload %s after %d retries", artifact.Name, repeats)
-			}
-			return nil
+			return upload(ctx, client, releaseID, artifact)
 		})
 	}
 	return g.Wait()
 }
 
 func upload(ctx *context.Context, client client.Client, releaseID string, artifact *artifact.Artifact) error {
-	file, err := os.Open(artifact.Path)
-	if err != nil {
-		return err
+	var repeats uint
+	what := func(try uint) error {
+		repeats = try + 1
+		file, err := os.Open(artifact.Path)
+		if err != nil {
+			return err
+		}
+		defer file.Close() // nolint: errcheck
+		log.WithField("file", file.Name()).WithField("name", artifact.Name).Info("uploading to release")
+		if err := client.Upload(ctx, releaseID, artifact, file); err != nil {
+			log.WithFields(log.Fields{
+				"try":      try,
+				"artifact": artifact.Name,
+			}).Warnf("failed to upload artifact, will retry")
+			return err
+		}
+		return nil
+	}
+	how := []func(uint, error) bool{
+		strategy.Limit(10),
+		strategy.Backoff(backoff.Linear(50 * time.Millisecond)),
+	}
+	if err := retry.Try(ctx, what, how...); err != nil {
+		return errors.Wrapf(err, "failed to upload %s after %d retries", artifact.Name, repeats)
+	}
+	return nil
+}
+
+func findFiles(ctx *context.Context) (map[string]string, error) {
+	var result = map[string]string{}
+	for _, extra := range ctx.Config.Release.ExtraFiles {
+		if extra.Glob != "" {
+			files, err := zglob.Glob(extra.Glob)
+			if err != nil {
+				return result, errors.Wrapf(err, "globbing failed for pattern %s", extra.Glob)
+			}
+			for _, file := range files {
+				info, err := os.Stat(file)
+				if err == nil && info.IsDir() {
+					log.Debugf("ignoring directory %s", file)
+					continue
+				}
+				var name = filepath.Base(file)
+				if old, ok := result[name]; ok {
+					log.Warnf("overriding %s with %s for name %s", old, file, name)
+				}
+				result[name] = file
+			}
+		}
 	}
-	defer file.Close() // nolint: errcheck
-	log.WithField("file", file.Name()).WithField("name", artifact.Name).Info("uploading to release")
-	return client.Upload(ctx, releaseID, artifact, file)
+	return result, nil
 }
diff --git a/internal/pipe/release/release_test.go b/internal/pipe/release/release_test.go
@@ -4,6 +4,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"testing"
 
@@ -104,6 +105,9 @@ func TestRunPipeWithIDsThenFilters(t *testing.T) {
 				Name:  "test",
 			},
 			IDs: []string{"foo"},
+			ExtraFiles: []config.ExtraFile{
+				{Glob: "./testdata/**/*"},
+			},
 		},
 	}
 	var ctx = context.New(config)
@@ -146,6 +150,9 @@ func TestRunPipeWithIDsThenFilters(t *testing.T) {
 	assert.True(t, client.UploadedFile)
 	assert.Contains(t, client.UploadedFileNames, "bin.deb")
 	assert.Contains(t, client.UploadedFileNames, "bin.tar.gz")
+	assert.Contains(t, client.UploadedFileNames, "release1.golden")
+	assert.Contains(t, client.UploadedFileNames, "release2.golden")
+	assert.Contains(t, client.UploadedFileNames, "f1")
 	assert.NotContains(t, client.UploadedFileNames, "filtered.deb")
 	assert.NotContains(t, client.UploadedFileNames, "filtered.tar.gz")
 }
@@ -219,6 +226,50 @@ func TestRunPipeUploadFailure(t *testing.T) {
 	assert.False(t, client.UploadedFile)
 }
 
+func TestRunPipeExtraFileNotFound(t *testing.T) {
+	var config = config.Project{
+		Release: config.Release{
+			GitHub: config.Repo{
+				Owner: "test",
+				Name:  "test",
+			},
+			ExtraFiles: []config.ExtraFile{
+				{Glob: "./testdata/release2.golden"},
+				{Glob: "./nope"},
+			},
+		},
+	}
+	var ctx = context.New(config)
+	ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"}
+	client := &DummyClient{}
+	assert.EqualError(t, doPublish(ctx, client), "globbing failed for pattern ./nope: file does not exist")
+	assert.True(t, client.CreatedRelease)
+	assert.False(t, client.UploadedFile)
+}
+
+func TestRunPipeExtraOverride(t *testing.T) {
+	var config = config.Project{
+		Release: config.Release{
+			GitHub: config.Repo{
+				Owner: "test",
+				Name:  "test",
+			},
+			ExtraFiles: []config.ExtraFile{
+				{Glob: "./testdata/**/*"},
+				{Glob: "./testdata/upload_same_name/f1"},
+			},
+		},
+	}
+	var ctx = context.New(config)
+	ctx.Git = context.GitInfo{CurrentTag: "v1.0.0"}
+	client := &DummyClient{}
+	assert.NoError(t, doPublish(ctx, client))
+	assert.True(t, client.CreatedRelease)
+	assert.True(t, client.UploadedFile)
+	assert.Contains(t, client.UploadedFileNames, "f1")
+	assert.True(t, strings.HasSuffix(client.UploadedFilePaths["f1"], "testdata/upload_same_name/f1"))
+}
+
 func TestRunPipeUploadRetry(t *testing.T) {
 	folder, err := ioutil.TempDir("", "goreleasertest")
 	assert.NoError(t, err)
@@ -470,6 +521,7 @@ type DummyClient struct {
 	CreatedRelease      bool
 	UploadedFile        bool
 	UploadedFileNames   []string
+	UploadedFilePaths   map[string]string
 	FailFirstUpload     bool
 	Lock                sync.Mutex
 }
@@ -489,6 +541,9 @@ func (client *DummyClient) CreateFile(ctx *context.Context, commitAuthor config.
 func (client *DummyClient) Upload(ctx *context.Context, releaseID string, artifact *artifact.Artifact, file *os.File) error {
 	client.Lock.Lock()
 	defer client.Lock.Unlock()
+	if client.UploadedFilePaths == nil {
+		client.UploadedFilePaths = map[string]string{}
+	}
 	// ensure file is read to better mimic real behavior
 	_, err := ioutil.ReadAll(file)
 	if err != nil {
@@ -503,5 +558,6 @@ func (client *DummyClient) Upload(ctx *context.Context, releaseID string, artifa
 	}
 	client.UploadedFile = true
 	client.UploadedFileNames = append(client.UploadedFileNames, artifact.Name)
+	client.UploadedFilePaths[artifact.Name] = artifact.Path
 	return nil
 }
diff --git a/internal/pipe/release/testdata/upload_same_name/f1 b/internal/pipe/release/testdata/upload_same_name/f1
diff --git a/internal/pipe/release/testdata/upload_same_name/f2/f1 b/internal/pipe/release/testdata/upload_same_name/f2/f1
diff --git a/pkg/config/config.go b/pkg/config/config.go
@@ -178,14 +178,20 @@ type Archive struct {
 
 // Release config used for the GitHub/GitLab release
 type Release struct {
-	GitHub       Repo     `yaml:",omitempty"`
-	GitLab       Repo     `yaml:",omitempty"`
-	Gitea        Repo     `yaml:",omitempty"`
-	Draft        bool     `yaml:",omitempty"`
-	Disable      bool     `yaml:",omitempty"`
-	Prerelease   string   `yaml:",omitempty"`
-	NameTemplate string   `yaml:"name_template,omitempty"`
-	IDs          []string `yaml:"ids,omitempty"`
+	GitHub       Repo        `yaml:",omitempty"`
+	GitLab       Repo        `yaml:",omitempty"`
+	Gitea        Repo        `yaml:",omitempty"`
+	Draft        bool        `yaml:",omitempty"`
+	Disable      bool        `yaml:",omitempty"`
+	Prerelease   string      `yaml:",omitempty"`
+	NameTemplate string      `yaml:"name_template,omitempty"`
+	IDs          []string    `yaml:"ids,omitempty"`
+	ExtraFiles   []ExtraFile `yaml:"extra_files,omitempty"`
+}
+
+// ExtraFile on a release
+type ExtraFile struct {
+	Glob string `yaml:"glob,omitempty"`
 }
 
 // NFPM config

diff --git a/www/content/release.md b/www/content/release.md
@@ -45,6 +45,15 @@ release:
   # GitHub.
   # Defaults to false.
   disable: true
+
+  # You can add extra pre-existing files to the release.
+  # The filename on the release will be the last part of the path (base). If
+  # another file with the same name exists, the latest one found will be used.
+  # Defaults to empty.
+  extra_files:
+    - glob: ./path/to/file.txt
+    - glob: ./glob/**/to/**/file/**/*
+    - glob: ./glob/foo/to/bar/file/foobar/override_from_previous
 ```
 
 Second, let's see what can be customized in the `release` section for GitLab.
@@ -73,6 +82,15 @@ release:
   # GitLab.
   # Defaults to false.
   disable: true
+
+  # You can add extra pre-existing files to the release.
+  # The filename on the release will be the last part of the path (base). If
+  # another file with the same name exists, the latest one found will be used.
+  # Defaults to empty.
+  extra_files:
+    - glob: ./path/to/file.txt
+    - glob: ./glob/**/to/**/file/**/*
+    - glob: ./glob/foo/to/bar/file/foobar/override_from_previous
 ```
 
 You can also configure the `release` section to upload to a [Gitea](https://gitea.io) instance:
@@ -99,6 +117,15 @@ release:
   # Gitea.
   # Defaults to false.
   disable: true
+
+  # You can add extra pre-existing files to the release.
+  # The filename on the release will be the last part of the path (base). If
+  # another file with the same name exists, the latest one found will be used.
+  # Defaults to empty.
+  extra_files:
+    - glob: ./path/to/file.txt
+    - glob: ./glob/**/to/**/file/**/*
+    - glob: ./glob/foo/to/bar/file/foobar/override_from_previous
 ```
 
 To enable uploading `tar.gz` and `checksums.txt` files you need to add the following to your Gitea config in `app.ini`: