From d7acf21f03d8e641fcf592043bf2baf934a678cc Mon Sep 17 00:00:00 2001 From: Carlos Alexandro Becker Date: Thu, 25 Aug 2022 02:15:37 -0300 Subject: [PATCH] fix: source archive add subfolders (#3343) `--add-file` adds only the `--prefix` joined with the basename of the added file, so, adding a folder like `vendor` would break havoc. this basically writes our own source archives (which are more compressed) and allows to add an entire folder easily, as well as other mappings as `archives` already supports. refs #3102 #2911 Signed-off-by: Carlos A Becker --- internal/pipe/sourcearchive/source.go | 92 ++++++++++------------ internal/pipe/sourcearchive/source_test.go | 85 ++++++++++++++++---- pkg/config/config.go | 10 +-- www/docs/customization/source.md | 17 +++- www/docs/static/schema.json | 2 +- 5 files changed, 133 insertions(+), 73 deletions(-) diff --git a/internal/pipe/sourcearchive/source.go b/internal/pipe/sourcearchive/source.go index 339d718e447..102713bb1d3 100644 --- a/internal/pipe/sourcearchive/source.go +++ b/internal/pipe/sourcearchive/source.go @@ -2,13 +2,17 @@ package sourcearchive import ( + "fmt" + "os" "path/filepath" + "strings" "github.com/caarlos0/log" "github.com/goreleaser/goreleaser/internal/archivefiles" "github.com/goreleaser/goreleaser/internal/artifact" "github.com/goreleaser/goreleaser/internal/git" "github.com/goreleaser/goreleaser/internal/tmpl" + "github.com/goreleaser/goreleaser/pkg/archive" "github.com/goreleaser/goreleaser/pkg/config" "github.com/goreleaser/goreleaser/pkg/context" ) @@ -33,31 +37,54 @@ func (Pipe) Run(ctx *context.Context) (err error) { filename := name + "." + ctx.Config.Source.Format path := filepath.Join(ctx.Config.Dist, filename) log.WithField("file", filename).Info("creating source archive") - args := []string{ - "archive", - "-o", path, - "--format", ctx.Config.Source.Format, + + out, err := git.Run(ctx, "ls-files") + if err != nil { + return fmt.Errorf("could not list source files: %w", err) } - if ctx.Config.Source.PrefixTemplate != "" { - prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate) - if err != nil { - return err - } - args = append(args, "--prefix", prefix) + prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate) + if err != nil { + return err } - files, err := evalFiles(ctx) + af, err := os.Create(path) + if err != nil { + return fmt.Errorf("could not create archive: %w", err) + } + defer af.Close() //nolint:errcheck + + arch, err := archive.New(af, ctx.Config.Source.Format) + if err != nil { + return err + } + + var ff []config.File + for _, f := range strings.Split(out, "\n") { + if strings.TrimSpace(f) == "" { + continue + } + ff = append(ff, config.File{ + Source: f, + }) + } + files, err := archivefiles.Eval(tmpl.New(ctx), append(ff, ctx.Config.Source.Files...)) if err != nil { return err } for _, f := range files { - args = append(args, "--add-file", f) + f.Destination = filepath.Join(prefix, f.Destination) + if err := arch.Add(f); err != nil { + return fmt.Errorf("could not add %q to archive: %w", f.Source, err) + } } - args = append(args, ctx.Git.FullCommit) - out, err := git.Clean(git.Run(ctx, args...)) - log.Debug(out) + if err := arch.Close(); err != nil { + return fmt.Errorf("could not close archive file: %w", err) + } + if err := af.Close(); err != nil { + return fmt.Errorf("could not close archive file: %w", err) + } ctx.Artifacts.Add(&artifact.Artifact{ Type: artifact.UploadableSourceArchive, @@ -70,41 +97,6 @@ func (Pipe) Run(ctx *context.Context) (err error) { return err } -// to reuse the archivefiles packages, we do something funky: -// - convert the []string to []config.File -// - eval it in archivefiles -// - convert it back to []string -// -// we also handle files already tracked, as if we add them again, -// they'll get duplicated in the archive. -func evalFiles(ctx *context.Context) ([]string, error) { - var files []config.File - for _, f := range ctx.Config.Source.Files { - files = append(files, config.File{ - Source: f, - }) - } - addFiles, err := archivefiles.Eval(tmpl.New(ctx), files) - if err != nil { - return nil, err - } - - var result []string - for _, f := range addFiles { - if isTracked(ctx, f.Source) { - continue - } - result = append(result, f.Source) - } - return result, nil -} - -// check if file is tracked, and, if it is we should not add it to the archive again. -func isTracked(ctx *context.Context, path string) bool { - _, err := git.Run(ctx, "ls-files", "--error-unmatch", path) - return err == nil -} - // Default sets the pipe defaults. func (Pipe) Default(ctx *context.Context) error { archive := &ctx.Config.Source diff --git a/internal/pipe/sourcearchive/source_test.go b/internal/pipe/sourcearchive/source_test.go index b28926f7418..a19b34562ab 100644 --- a/internal/pipe/sourcearchive/source_test.go +++ b/internal/pipe/sourcearchive/source_test.go @@ -27,6 +27,8 @@ func TestArchive(t *testing.T) { testlib.GitCommit(t, "feat: first") require.NoError(t, os.WriteFile("added-later.txt", []byte("this file was added later"), 0o655)) require.NoError(t, os.WriteFile("ignored.md", []byte("never added"), 0o655)) + require.NoError(t, os.MkdirAll("subfolder", 0o755)) + require.NoError(t, os.WriteFile("subfolder/file.md", []byte("a file within a folder, added later"), 0o655)) ctx := context.New(config.Project{ ProjectName: "foo", @@ -35,8 +37,9 @@ func TestArchive(t *testing.T) { Format: format, Enabled: true, PrefixTemplate: "{{ .ProjectName }}-{{ .Version }}/", - Files: []string{ - "*.txt", + Files: []config.File{ + {Source: "*.txt"}, + {Source: "subfolder/*"}, }, }, }) @@ -65,22 +68,13 @@ func TestArchive(t *testing.T) { return } - f, err := os.Open(path) - require.NoError(t, err) - z, err := zip.NewReader(f, stat.Size()) - require.NoError(t, err) - - var paths []string - for _, zf := range z.File { - paths = append(paths, zf.Name) - } - require.Equal(t, []string{ - "foo-1.0.0/", + require.ElementsMatch(t, []string{ "foo-1.0.0/README.md", "foo-1.0.0/code.py", "foo-1.0.0/code.txt", "foo-1.0.0/added-later.txt", - }, paths) + "foo-1.0.0/subfolder/file.md", + }, lsZip(t, path)) }) } } @@ -96,7 +90,7 @@ func TestInvalidFormat(t *testing.T) { }, }) require.NoError(t, Pipe{}.Default(ctx)) - require.EqualError(t, Pipe{}.Run(ctx), "fatal: Unknown archive format '7z'") + require.EqualError(t, Pipe{}.Run(ctx), "invalid archive format: 7z") } func TestDefault(t *testing.T) { @@ -115,7 +109,45 @@ func TestInvalidNameTemplate(t *testing.T) { NameTemplate: "{{ .foo }-asdda", }, }) - require.EqualError(t, Pipe{}.Run(ctx), "template: tmpl:1: unexpected \"}\" in operand") + testlib.RequireTemplateError(t, Pipe{}.Run(ctx)) +} + +func TestInvalidInvalidFileTemplate(t *testing.T) { + testlib.Mktmp(t) + require.NoError(t, os.Mkdir("dist", 0o744)) + + testlib.GitInit(t) + require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655)) + testlib.GitAdd(t) + testlib.GitCommit(t, "feat: first") + + ctx := context.New(config.Project{ + ProjectName: "foo", + Dist: "dist", + Source: config.Source{ + Format: "tar.gz", + Enabled: true, + Files: []config.File{ + {Source: "{{.Test}"}, + }, + }, + }) + ctx.Git.FullCommit = "HEAD" + ctx.Version = "1.0.0" + require.NoError(t, Pipe{}.Default(ctx)) + testlib.RequireTemplateError(t, Pipe{}.Run(ctx)) +} + +func TestInvalidPrefixTemplate(t *testing.T) { + ctx := context.New(config.Project{ + Dist: t.TempDir(), + Source: config.Source{ + Enabled: true, + PrefixTemplate: "{{ .ProjectName }/", + }, + }) + require.NoError(t, Pipe{}.Default(ctx)) + testlib.RequireTemplateError(t, Pipe{}.Run(ctx)) } func TestDisabled(t *testing.T) { @@ -136,3 +168,24 @@ func TestSkip(t *testing.T) { require.False(t, Pipe{}.Skip(ctx)) }) } + +func TestString(t *testing.T) { + require.NotEmpty(t, Pipe{}.String()) +} + +func lsZip(tb testing.TB, path string) []string { + tb.Helper() + + stat, err := os.Stat(path) + require.NoError(tb, err) + f, err := os.Open(path) + require.NoError(tb, err) + z, err := zip.NewReader(f, stat.Size()) + require.NoError(tb, err) + + var paths []string + for _, zf := range z.File { + paths = append(paths, zf.Name) + } + return paths +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 6dcde36c472..0bab72524f3 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -876,11 +876,11 @@ type Publisher struct { // Source configuration. type Source struct { - NameTemplate string `yaml:"name_template,omitempty" json:"name_template,omitempty"` - Format string `yaml:"format,omitempty" json:"format,omitempty"` - Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` - PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"` - Files []string `yaml:"files,omitempty" json:"files,omitempty"` + NameTemplate string `yaml:"name_template,omitempty" json:"name_template,omitempty"` + Format string `yaml:"format,omitempty" json:"format,omitempty"` + Enabled bool `yaml:"enabled,omitempty" json:"enabled,omitempty"` + PrefixTemplate string `yaml:"prefix_template,omitempty" json:"prefix_template,omitempty"` + Files []File `yaml:"files,omitempty" json:"files,omitempty"` } // Project includes all project configuration. diff --git a/www/docs/customization/source.md b/www/docs/customization/source.md index ef1f51815a6..bfec2703804 100644 --- a/www/docs/customization/source.md +++ b/www/docs/customization/source.md @@ -25,7 +25,6 @@ source: prefix_template: '{{ .ProjectName }}-{{ .Version }}/' # Additional files/template/globs you want to add to the source archive. - # Will use --add-file of git-archive. # Defaults to empty. files: - LICENSE.txt @@ -34,6 +33,22 @@ source: - docs/* - design/*.png - templates/**/* + # a more complete example, check the globbing deep dive below + - src: '*.md' + dst: docs + # Strip parent folders when adding files to the archive. + # Default: false + strip_parent: true + # File info. + # Not all fields are supported by all formats available formats. + # Defaults to the file info of the actual file if not provided. + info: + owner: root + group: root + mode: 0644 + # format is `time.RFC3339Nano` + mtime: 2008-01-02T15:04:05Z + ``` !!! tip diff --git a/www/docs/static/schema.json b/www/docs/static/schema.json index 4ab5dbdfc76..24b278aaf2c 100644 --- a/www/docs/static/schema.json +++ b/www/docs/static/schema.json @@ -2382,7 +2382,7 @@ }, "files": { "items": { - "type": "string" + "$ref": "#/$defs/File" }, "type": "array" }