Use cacheitem to restore HTTP cache.

cli/internal/cache/cache_http.go

@@ -20,6 +20,7 @@ import (
 	"github.com/DataDog/zstd"
 
 	"github.com/vercel/turbo/cli/internal/analytics"
+	"github.com/vercel/turbo/cli/internal/cacheitem"
 	"github.com/vercel/turbo/cli/internal/tarpatch"
 	"github.com/vercel/turbo/cli/internal/turbopath"
 )
@@ -251,102 +252,9 @@ func (cache *httpCache) retrieve(hash string) (bool, []turbopath.AnchoredSystemP
 	return true, files, duration, nil
 }
 
-// restoreTar returns posix-style repo-relative paths of the files it
-// restored. In the future, these should likely be repo-relative system paths
-// so that they are suitable for being fed into cache.Put for other caches.
-// For now, I think this is working because windows also accepts /-delimited paths.
 func restoreTar(root turbopath.AbsoluteSystemPath, reader io.Reader) ([]turbopath.AnchoredSystemPath, error) {
-	files := []turbopath.AnchoredSystemPath{}
-	missingLinks := []*tar.Header{}
-	zr := zstd.NewReader(reader)
-	var closeError error
-	defer func() { closeError = zr.Close() }()
-	tr := tar.NewReader(zr)
-	for {
-		hdr, err := tr.Next()
-		if err != nil {
-			if err == io.EOF {
-				for _, link := range missingLinks {
-					err := restoreSymlink(root, link, true)
-					if err != nil {
-						return nil, err
-					}
-				}
-
-				return files, closeError
-			}
-			return nil, err
-		}
-		// hdr.Name is always a posix-style path
-		// FIXME: THIS IS A BUG.
-		restoredName := turbopath.AnchoredUnixPath(hdr.Name)
-		files = append(files, restoredName.ToSystemPath())
-		filename := restoredName.ToSystemPath().RestoreAnchor(root)
-		if isChild, err := root.ContainsPath(filename); err != nil {
-			return nil, err
-		} else if !isChild {
-			return nil, fmt.Errorf("cannot untar file to %v", filename)
-		}
-		switch hdr.Typeflag {
-		case tar.TypeDir:
-			if err := filename.MkdirAll(0775); err != nil {
-				return nil, err
-			}
-		case tar.TypeReg:
-			if dir := filename.Dir(); dir != "." {
-				if err := dir.MkdirAll(0775); err != nil {
-					return nil, err
-				}
-			}
-			if f, err := filename.OpenFile(os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.FileMode(hdr.Mode)); err != nil {
-				return nil, err
-			} else if _, err := io.Copy(f, tr); err != nil {
-				return nil, err
-			} else if err := f.Close(); err != nil {
-				return nil, err
-			}
-		case tar.TypeSymlink:
-			if err := restoreSymlink(root, hdr, false); errors.Is(err, errNonexistentLinkTarget) {
-				missingLinks = append(missingLinks, hdr)
-			} else if err != nil {
-				return nil, err
-			}
-		default:
-			log.Printf("Unhandled file type %d for %s", hdr.Typeflag, hdr.Name)
-		}
-	}
-}
-
-var errNonexistentLinkTarget = errors.New("the link target does not exist")
-
-func restoreSymlink(root turbopath.AbsoluteSystemPath, hdr *tar.Header, allowNonexistentTargets bool) error {
-	// Note that hdr.Linkname is really the link target
-	relativeLinkTarget := filepath.FromSlash(hdr.Linkname)
-	linkFilename := root.UntypedJoin(hdr.Name)
-	if err := linkFilename.EnsureDir(); err != nil {
-		return err
-	}
-
-	// TODO: check if this is an absolute path, or if we even care
-	linkTarget := linkFilename.Dir().UntypedJoin(relativeLinkTarget)
-	if _, err := linkTarget.Lstat(); err != nil {
-		if os.IsNotExist(err) {
-			if !allowNonexistentTargets {
-				return errNonexistentLinkTarget
-			}
-			// if we're allowing nonexistent link targets, proceed to creating the link
-		} else {
-			return err
-		}
-	}
-	// Ensure that the link we're about to create doesn't already exist
-	if err := linkFilename.Remove(); err != nil && !errors.Is(err, os.ErrNotExist) {
-		return err
-	}
-	if err := linkFilename.Symlink(relativeLinkTarget); err != nil {
-		return err
-	}
-	return nil
+	cache := cacheitem.FromReader(reader, true)
+	return cache.Restore(root)
 }
 
 func (cache *httpCache) Clean(_ turbopath.AbsoluteSystemPath) {

cli/internal/cache/cache_http_test.go

@@ -81,7 +81,7 @@ func makeValidTar(t *testing.T) *bytes.Buffer {
 	// my-pkg
 	h := &tar.Header{
 		Name:     "my-pkg/",
-		Mode:     int64(0644),
+		Mode:     int64(0755),
 		Typeflag: tar.TypeDir,
 	}
 	if err := tw.WriteHeader(h); err != nil {
@@ -182,7 +182,7 @@ func TestRestoreTar(t *testing.T) {
 
 	expectedFiles := []turbopath.AnchoredSystemPath{
 		turbopath.AnchoredUnixPath("extra-file").ToSystemPath(),
-		turbopath.AnchoredUnixPath("my-pkg/").ToSystemPath(),
+		turbopath.AnchoredUnixPath("my-pkg").ToSystemPath(),
 		turbopath.AnchoredUnixPath("my-pkg/some-file").ToSystemPath(),
 		turbopath.AnchoredUnixPath("my-pkg/link-to-extra-file").ToSystemPath(),
 		turbopath.AnchoredUnixPath("my-pkg/broken-link").ToSystemPath(),

cli/internal/cacheitem/cacheitem.go

@@ -7,7 +7,6 @@ import (
 	"crypto/sha512"
 	"errors"
 	"io"
-	"os"
 
 	"github.com/vercel/turbo/cli/internal/turbopath"
 )
@@ -32,7 +31,7 @@ type CacheItem struct {
 	tw         *tar.Writer
 	zw         io.WriteCloser
 	fileBuffer *bufio.Writer
-	handle     *os.File
+	handle     io.Reader
 	compressed bool
 }
 
@@ -57,9 +56,14 @@ func (ci *CacheItem) Close() error {
 	}
 
 	if ci.handle != nil {
-		if err := ci.handle.Close(); err != nil {
-			return err
+		closer, isCloser := ci.handle.(io.Closer)
+
+		if isCloser {
+			if err := closer.Close(); err != nil {
+				return err
+			}
 		}
+
 	}
 
 	return nil

cli/internal/cacheitem/create.go

@@ -36,7 +36,12 @@ func Create(path turbopath.AbsoluteSystemPath) (*CacheItem, error) {
 // Wires all the writers end-to-end:
 // tar.Writer -> zstd.Writer -> fileBuffer -> file
 func (ci *CacheItem) init() {
-	fileBuffer := bufio.NewWriterSize(ci.handle, 2^20) // Flush to disk in 1mb chunks.
+	writer, isWriter := ci.handle.(io.Writer)
+	if !isWriter {
+		panic("can't write to this cache item")
+	}
+
+	fileBuffer := bufio.NewWriterSize(writer, 2^20) // Flush to disk in 1mb chunks.
 
 	var tw *tar.Writer
 	if ci.compressed {

cli/internal/cacheitem/restore.go

@@ -14,6 +14,14 @@ import (
 	"github.com/vercel/turbo/cli/internal/turbopath"
 )
 
+// FromReader returns an existing CacheItem at the specified path.
+func FromReader(reader io.Reader, compressed bool) *CacheItem {
+	return &CacheItem{
+		handle:     reader,
+		compressed: compressed,
+	}
+}
+
 // Open returns an existing CacheItem at the specified path.
 func Open(path turbopath.AbsoluteSystemPath) (*CacheItem, error) {
 	handle, err := sequential.OpenFile(path.ToString(), os.O_RDONLY, 0777)