fix: use git-archive under the hood (#3904)

This reverts back to using `git archive` for the source archives... but
will keep supporting extra files.

##### How it works:

Basically, we run `git archive` as before.
Then, we make a backup of the generated archive, and create a new one
copying by reading from the backup and writing into the new one.
Finally, we write the extra files to the new one as well.

This only happens if the configuration does have extra files, otherwise,
just the simple `git archive` will be run.

PS: we can't just append to the archive because weird tar format
paddings et al.

---------

Signed-off-by: Carlos Alexandro Becker <caarlos0@users.noreply.github.com>
Signed-off-by: Carlos A Becker <caarlos0@users.noreply.github.com>

internal/pipe/archive/archive.go

@@ -317,10 +317,6 @@ type EnhancedArchive struct {
 func (d EnhancedArchive) Add(f config.File) error {
 	name := strings.ReplaceAll(filepath.Join(d.wrap, f.Destination), "\\", "/")
 	log.Debugf("adding file: %s as %s", f.Source, name)
-	if _, ok := d.files[f.Destination]; ok {
-		return fmt.Errorf("file %s already exists in the archive", f.Destination)
-	}
-	d.files[f.Destination] = name
 	ff := config.File{
 		Source:      f.Source,
 		Destination: name,

internal/pipe/archive/archive_meta_test.go

@@ -5,6 +5,7 @@ import (
 	"testing"
 
 	"github.com/goreleaser/goreleaser/internal/testctx"
+	"github.com/goreleaser/goreleaser/internal/testlib"
 	"github.com/goreleaser/goreleaser/pkg/config"
 	"github.com/stretchr/testify/require"
 )
@@ -30,7 +31,7 @@ func TestMeta(t *testing.T) {
 		require.Equal(
 			t,
 			[]string{"testdata/a/a.txt", "testdata/a/b/a.txt", "testdata/a/b/c/d.txt"},
-			tarFiles(t, filepath.Join(dist, "foo.tar.gz")),
+			testlib.LsArchive(t, filepath.Join(dist, "foo.tar.gz"), "tar.gz"),
 		)
 	})
 

internal/pipe/archive/archive_test.go

@@ -2,10 +2,10 @@ package archive
 
 import (
 	"archive/tar"
-	"archive/zip"
 	"compress/gzip"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"testing"
@@ -235,7 +235,7 @@ func TestRunPipe(t *testing.T) {
 							"foo/bar/foobar/blah.txt",
 							expectBin,
 						},
-						tarFiles(t, filepath.Join(dist, name)),
+						testlib.LsArchive(t, filepath.Join(dist, name), "tar.gz"),
 					)
 
 					header := tarInfo(t, filepath.Join(dist, name), expectBin)
@@ -252,7 +252,7 @@ func TestRunPipe(t *testing.T) {
 						"foo/bar/foobar/blah.txt",
 						expectBin + ".exe",
 					},
-					zipFiles(t, filepath.Join(dist, "foobar_0.0.1_windows_amd64.zip")),
+					testlib.LsArchive(t, filepath.Join(dist, "foobar_0.0.1_windows_amd64.zip"), "zip"),
 				)
 			}
 		})
@@ -355,21 +355,6 @@ func TestRunPipeNoBinaries(t *testing.T) {
 	require.NoError(t, Pipe{}.Run(ctx))
 }
 
-func zipFiles(t *testing.T, path string) []string {
-	t.Helper()
-	f, err := os.Open(path)
-	require.NoError(t, err)
-	info, err := f.Stat()
-	require.NoError(t, err)
-	r, err := zip.NewReader(f, info.Size())
-	require.NoError(t, err)
-	paths := make([]string, len(r.File))
-	for i, zf := range r.File {
-		paths[i] = zf.Name
-	}
-	return paths
-}
-
 func tarInfo(t *testing.T, path, name string) *tar.Header {
 	t.Helper()
 	f, err := os.Open(path)
@@ -391,27 +376,6 @@ func tarInfo(t *testing.T, path, name string) *tar.Header {
 	return nil
 }
 
-func tarFiles(t *testing.T, path string) []string {
-	t.Helper()
-	f, err := os.Open(path)
-	require.NoError(t, err)
-	defer f.Close()
-	gr, err := gzip.NewReader(f)
-	require.NoError(t, err)
-	defer gr.Close()
-	r := tar.NewReader(gr)
-	var paths []string
-	for {
-		next, err := r.Next()
-		if err == io.EOF {
-			break
-		}
-		require.NoError(t, err)
-		paths = append(paths, next.Name)
-	}
-	return paths
-}
-
 func TestRunPipeBinary(t *testing.T) {
 	folder := testlib.Mktmp(t)
 	dist := filepath.Join(folder, "dist")
@@ -742,22 +706,11 @@ func TestRunPipeWrap(t *testing.T) {
 	require.Len(t, archives, 1)
 	require.Equal(t, "foo_macOS", artifact.ExtraOr(*archives[0], artifact.ExtraWrappedIn, ""))
 
-	// Check archive contents
-	f, err = os.Open(filepath.Join(dist, "foo.tar.gz"))
-	require.NoError(t, err)
-	defer func() { require.NoError(t, f.Close()) }()
-	gr, err := gzip.NewReader(f)
-	require.NoError(t, err)
-	defer func() { require.NoError(t, gr.Close()) }()
-	r := tar.NewReader(gr)
-	for _, n := range []string{"README.md", "mybin"} {
-		h, err := r.Next()
-		if err == io.EOF {
-			break
-		}
-		require.NoError(t, err)
-		require.Equal(t, filepath.Join("foo_macOS", n), h.Name)
-	}
+	require.ElementsMatch(
+		t,
+		[]string{"foo_macOS/README.md", "foo_macOS/mybin"},
+		testlib.LsArchive(t, filepath.Join(dist, "foo.tar.gz"), "tar.gz"),
+	)
 }
 
 func TestDefault(t *testing.T) {
@@ -999,10 +952,10 @@ func TestDuplicateFilesInsideArchive(t *testing.T) {
 		Source:      ff.Name(),
 		Destination: "foo",
 	}))
-	require.EqualError(t, a.Add(config.File{
+	require.ErrorIs(t, a.Add(config.File{
 		Source:      ff.Name(),
 		Destination: "foo",
-	}), "file foo already exists in the archive")
+	}), fs.ErrExist)
 }
 
 func TestWrapInDirectory(t *testing.T) {
@@ -1068,7 +1021,7 @@ func TestArchive_globbing(t *testing.T) {
 		})
 
 		require.NoError(t, Pipe{}.Run(ctx))
-		require.Equal(t, append(expected, "foobin"), tarFiles(t, filepath.Join(dist, "foo.tar.gz")))
+		require.Equal(t, append(expected, "foobin"), testlib.LsArchive(t, filepath.Join(dist, "foo.tar.gz"), "tar.gz"))
 	}
 
 	t.Run("exact src file", func(t *testing.T) {

internal/pipe/sourcearchive/source.go

@@ -5,16 +5,15 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/caarlos0/log"
 	"github.com/goreleaser/goreleaser/internal/archivefiles"
 	"github.com/goreleaser/goreleaser/internal/artifact"
 	"github.com/goreleaser/goreleaser/internal/deprecate"
+	"github.com/goreleaser/goreleaser/internal/gio"
 	"github.com/goreleaser/goreleaser/internal/git"
 	"github.com/goreleaser/goreleaser/internal/tmpl"
 	"github.com/goreleaser/goreleaser/pkg/archive"
-	"github.com/goreleaser/goreleaser/pkg/config"
 	"github.com/goreleaser/goreleaser/pkg/context"
 )
 
@@ -30,49 +29,70 @@ func (Pipe) Skip(ctx *context.Context) bool {
 }
 
 // Run the pipe.
-func (Pipe) Run(ctx *context.Context) (err error) {
+func (Pipe) Run(ctx *context.Context) error {
+	format := ctx.Config.Source.Format
+	if format != "zip" && format != "tar" && format != "tgz" && format != "tar.gz" {
+		return fmt.Errorf("invalid source archive format: %s", format)
+	}
 	name, err := tmpl.New(ctx).Apply(ctx.Config.Source.NameTemplate)
 	if err != nil {
 		return err
 	}
-	filename := name + "." + ctx.Config.Source.Format
+	filename := name + "." + format
 	path := filepath.Join(ctx.Config.Dist, filename)
 	log.WithField("file", filename).Info("creating source archive")
+	args := []string{
+		"archive",
+		"-o", path,
+	}
 
-	out, err := git.Run(ctx, "ls-files")
-	if err != nil {
-		return fmt.Errorf("could not list source files: %w", err)
+	prefix := ""
+	if ctx.Config.Source.PrefixTemplate != "" {
+		pt, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
+		if err != nil {
+			return err
+		}
+		prefix = pt
+		args = append(args, "--prefix", prefix)
 	}
+	args = append(args, ctx.Git.FullCommit)
 
-	prefix, err := tmpl.New(ctx).Apply(ctx.Config.Source.PrefixTemplate)
-	if err != nil {
+	if _, err := git.Clean(git.Run(ctx, args...)); err != nil {
 		return err
 	}
 
-	af, err := os.Create(path)
+	if len(ctx.Config.Source.Files) == 0 {
+		return nil
+	}
+
+	oldPath := path + ".bkp"
+	if err := gio.Copy(path, oldPath); err != nil {
+		return fmt.Errorf("failed make a backup of %q: %w", path, err)
+	}
+
+	// i could spend a lot of time trying to figure out how to append to a tar,
+	// tgz and zip file... but... this seems easy enough :)
+	of, err := os.Open(oldPath)
 	if err != nil {
-		return fmt.Errorf("could not create archive: %w", err)
+		return fmt.Errorf("could not open %q: %w", oldPath, err)
+	}
+	defer of.Close()
+
+	af, err := os.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0o644)
+	if err != nil {
+		return fmt.Errorf("could not open archive: %w", err)
 	}
 	defer af.Close() //nolint:errcheck
 
-	arch, err := archive.New(af, ctx.Config.Source.Format)
+	arch, err := archive.Copying(of, af, format)
 	if err != nil {
 		return err
 	}
 
-	var ff []config.File
-	for _, f := range strings.Split(out, "\n") {
-		if strings.TrimSpace(f) == "" {
-			continue
-		}
-		ff = append(ff, config.File{
-			Source: f,
-		})
-	}
 	files, err := archivefiles.Eval(
 		tmpl.New(ctx),
 		ctx.Config.Source.RLCP,
-		append(ff, ctx.Config.Source.Files...),
+		ctx.Config.Source.Files,
 	)
 	if err != nil {
 		return err
@@ -96,7 +116,7 @@ func (Pipe) Run(ctx *context.Context) (err error) {
 		Name: filename,
 		Path: path,
 		Extra: map[string]interface{}{
-			artifact.ExtraFormat: ctx.Config.Source.Format,
+			artifact.ExtraFormat: format,
 		},
 	})
 	return err

internal/pipe/sourcearchive/source_test.go

@@ -1,7 +1,6 @@
 package sourcearchive
 
 import (
-	"archive/zip"
 	"os"
 	"path/filepath"
 	"testing"
@@ -21,13 +20,14 @@ func TestArchive(t *testing.T) {
 			require.NoError(t, os.Mkdir("dist", 0o744))
 
 			testlib.GitInit(t)
-			require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655))
+			require.NoError(t, os.WriteFile("code.rb", []byte("not really code"), 0o655))
 			require.NoError(t, os.WriteFile("code.py", []byte("print 1"), 0o655))
 			require.NoError(t, os.WriteFile("README.md", []byte("# my dope fake project"), 0o655))
 			testlib.GitAdd(t)
 			testlib.GitCommit(t, "feat: first")
 			require.NoError(t, os.WriteFile("added-later.txt", []byte("this file was added later"), 0o655))
 			require.NoError(t, os.WriteFile("ignored.md", []byte("never added"), 0o655))
+			require.NoError(t, os.WriteFile("code.txt", []byte("not really code"), 0o655))
 			require.NoError(t, os.MkdirAll("subfolder", 0o755))
 			require.NoError(t, os.WriteFile("subfolder/file.md", []byte("a file within a folder, added later"), 0o655))
 
@@ -67,17 +67,22 @@ func TestArchive(t *testing.T) {
 			require.NoError(t, err)
 			require.Greater(t, stat.Size(), int64(100))
 
-			if format != "zip" {
-				return
-			}
-
-			require.ElementsMatch(t, []string{
+			expected := []string{
+				"foo-1.0.0/",
 				"foo-1.0.0/README.md",
 				"foo-1.0.0/code.py",
+				"foo-1.0.0/code.rb",
 				"foo-1.0.0/code.txt",
 				"foo-1.0.0/added-later.txt",
 				"foo-1.0.0/subfolder/file.md",
-			}, lsZip(t, path))
+			}
+
+			// zips wont have the parent dir
+			if format == "zip" {
+				expected = expected[1:]
+			}
+
+			require.ElementsMatch(t, expected, testlib.LsArchive(t, path, format))
 		})
 	}
 }
@@ -93,7 +98,7 @@ func TestInvalidFormat(t *testing.T) {
 		},
 	})
 	require.NoError(t, Pipe{}.Default(ctx))
-	require.EqualError(t, Pipe{}.Run(ctx), "invalid archive format: 7z")
+	require.EqualError(t, Pipe{}.Run(ctx), "invalid source archive format: 7z")
 }
 
 func TestDefault(t *testing.T) {
@@ -112,6 +117,7 @@ func TestInvalidNameTemplate(t *testing.T) {
 			NameTemplate: "{{ .foo }-asdda",
 		},
 	})
+	require.NoError(t, Pipe{}.Default(ctx))
 	testlib.RequireTemplateError(t, Pipe{}.Run(ctx))
 }
 
@@ -175,20 +181,3 @@ func TestSkip(t *testing.T) {
 func TestString(t *testing.T) {
 	require.NotEmpty(t, Pipe{}.String())
 }
-
-func lsZip(tb testing.TB, path string) []string {
-	tb.Helper()
-
-	stat, err := os.Stat(path)
-	require.NoError(tb, err)
-	f, err := os.Open(path)
-	require.NoError(tb, err)
-	z, err := zip.NewReader(f, stat.Size())
-	require.NoError(tb, err)
-
-	var paths []string
-	for _, zf := range z.File {
-		paths = append(paths, zf.Name)
-	}
-	return paths
-}