chore: move lockfile ffi boundary

cli/internal/context/context.go

@@ -17,7 +17,6 @@ import (
 	"github.com/vercel/turbo/cli/internal/workspace"
 
 	"github.com/Masterminds/semver"
-	mapset "github.com/deckarep/golang-set"
 	"github.com/pyr-sh/dag"
 	"golang.org/x/sync/errgroup"
 )
@@ -179,12 +178,6 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
 	}
 	c.PackageManager = packageManager
 
-	if lockfile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil {
-		warnings.append(err)
-	} else {
-		c.Lockfile = lockfile
-	}
-
 	if err := c.resolveWorkspaceRootDeps(rootPackageJSON, &warnings); err != nil {
 		// TODO(Gaspar) was this the intended return error?
 		return nil, fmt.Errorf("could not resolve workspaces: %w", err)
@@ -214,10 +207,9 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
 	}
 	populateGraphWaitGroup := &errgroup.Group{}
 	for _, pkg := range c.WorkspaceInfos.PackageJSONs {
-		pkg := pkg
-		populateGraphWaitGroup.Go(func() error {
-			return c.populateWorkspaceGraphForPackageJSON(pkg, rootpath, pkg.Name, &warnings)
-		})
+		if err := c.populateWorkspaceGraphForPackageJSON(pkg, rootpath, pkg.Name, &warnings); err != nil {
+			return nil, err
+		}
 	}
 
 	if err := populateGraphWaitGroup.Wait(); err != nil {
@@ -232,6 +224,25 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
 	}
 	c.WorkspaceInfos.PackageJSONs[util.RootPkgName] = rootPackageJSON
 
+	if lockFile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil {
+		warnings.append(err)
+		rootPackageJSON.TransitiveDeps = []lockfile.Package{}
+		rootPackageJSON.ExternalDepsHash = ""
+	} else {
+		c.Lockfile = lockFile
+		if closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), c.Lockfile); err != nil {
+			warnings.append(err)
+		} else {
+			for _, pkg := range c.WorkspaceInfos.PackageJSONs {
+				if closure, ok := closures[pkg.Dir.ToUnixPath()]; ok {
+					if err := pkg.SetExternalDeps(closure); err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
+	}
+
 	return c, warnings.errorOrNil()
 }
 
@@ -247,33 +258,6 @@ func (c *Context) resolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON, warn
 	for dep, version := range pkg.Dependencies {
 		pkg.UnresolvedExternalDeps[dep] = version
 	}
-	if c.Lockfile != nil {
-		depSet, err := lockfile.TransitiveClosure(
-			pkg.Dir.ToUnixPath(),
-			pkg.UnresolvedExternalDeps,
-			c.Lockfile,
-		)
-		if err != nil {
-			warnings.append(err)
-			// Return early to skip using results of incomplete dep graph resolution
-			return nil
-		}
-		pkg.TransitiveDeps = make([]lockfile.Package, 0, depSet.Cardinality())
-		for _, v := range depSet.ToSlice() {
-			dep := v.(lockfile.Package)
-			pkg.TransitiveDeps = append(pkg.TransitiveDeps, dep)
-		}
-		sort.Sort(lockfile.ByKey(pkg.TransitiveDeps))
-		hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps)
-		if err != nil {
-			return err
-		}
-		pkg.ExternalDepsHash = hashOfExternalDeps
-	} else {
-		pkg.TransitiveDeps = []lockfile.Package{}
-		pkg.ExternalDepsHash = ""
-	}
-
 	return nil
 }
 
@@ -282,8 +266,6 @@ func (c *Context) resolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON, warn
 // that are not within the monorepo. The vertexName is used to override the package name in the graph.
 // This can happen when adding the root package, which can have an arbitrary name.
 func (c *Context) populateWorkspaceGraphForPackageJSON(pkg *fs.PackageJSON, rootpath string, vertexName string, warnings *Warnings) error {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
 	depMap := make(map[string]string)
 	internalDepsSet := make(dag.Set)
 	externalUnresolvedDepsSet := make(dag.Set)
@@ -326,37 +308,18 @@ func (c *Context) populateWorkspaceGraphForPackageJSON(pkg *fs.PackageJSON, root
 		}
 	}
 
-	externalDeps, err := lockfile.TransitiveClosure(
-		pkg.Dir.ToUnixPath(),
-		pkg.UnresolvedExternalDeps,
-		c.Lockfile,
-	)
-	if err != nil {
-		warnings.append(err)
-		// reset external deps to original state
-		externalDeps = mapset.NewSet()
-	}
-
 	// when there are no internal dependencies, we need to still add these leafs to the graph
 	if internalDepsSet.Len() == 0 {
 		c.WorkspaceGraph.Connect(dag.BasicEdge(pkg.Name, core.ROOT_NODE_NAME))
 	}
-	pkg.TransitiveDeps = make([]lockfile.Package, 0, externalDeps.Cardinality())
-	for _, dependency := range externalDeps.ToSlice() {
-		dependency := dependency.(lockfile.Package)
-		pkg.TransitiveDeps = append(pkg.TransitiveDeps, dependency)
-	}
+
 	pkg.InternalDeps = make([]string, 0, internalDepsSet.Len())
 	for _, v := range internalDepsSet.List() {
 		pkg.InternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%v", v))
 	}
+
 	sort.Strings(pkg.InternalDeps)
-	sort.Sort(lockfile.ByKey(pkg.TransitiveDeps))
-	hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps)
-	if err != nil {
-		return err
-	}
-	pkg.ExternalDepsHash = hashOfExternalDeps
+
 	return nil
 }
 
@@ -387,6 +350,14 @@ func (c *Context) parsePackageJSON(repoRoot turbopath.AbsoluteSystemPath, pkgJSO
 	return nil
 }
 
+func (c *Context) externalWorkspaceDeps() map[turbopath.AnchoredUnixPath]map[string]string {
+	workspaces := make(map[turbopath.AnchoredUnixPath]map[string]string, len(c.WorkspaceInfos.PackageJSONs))
+	for _, pkg := range c.WorkspaceInfos.PackageJSONs {
+		workspaces[pkg.Dir.ToUnixPath()] = pkg.UnresolvedExternalDeps
+	}
+	return workspaces
+}
+
 // InternalDependencies finds all dependencies required by the slice of starting
 // packages, as well as the starting packages themselves.
 func (c *Context) InternalDependencies(start []string) ([]string, error) {
@@ -424,13 +395,14 @@ func (c *Context) ChangedPackages(previousLockfile lockfile.Lockfile) ([]string,
 		return nil, fmt.Errorf("Cannot detect changed packages without previous and current lockfile")
 	}
 
+	closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), previousLockfile)
+	if err != nil {
+		return nil, err
+	}
+
 	didPackageChange := func(pkgName string, pkg *fs.PackageJSON) bool {
-		previousDeps, err := lockfile.TransitiveClosure(
-			pkg.Dir.ToUnixPath(),
-			pkg.UnresolvedExternalDeps,
-			previousLockfile,
-		)
-		if err != nil || previousDeps.Cardinality() != len(pkg.TransitiveDeps) {
+		previousDeps, ok := closures[pkg.Dir.ToUnixPath()]
+		if !ok || previousDeps.Cardinality() != len(pkg.TransitiveDeps) {
 			return true
 		}
 

cli/internal/fs/package_json.go

@@ -3,8 +3,10 @@ package fs
 import (
 	"bytes"
 	"encoding/json"
+	"sort"
 	"sync"
 
+	mapset "github.com/deckarep/golang-set"
 	"github.com/vercel/turbo/cli/internal/lockfile"
 	"github.com/vercel/turbo/cli/internal/turbopath"
 )
@@ -120,6 +122,24 @@ func MarshalPackageJSON(pkgJSON *PackageJSON) ([]byte, error) {
 	return b.Bytes(), nil
 }
 
+// SetExternalDeps sets TransitiveDeps and populates ExternalDepsHash
+func (p *PackageJSON) SetExternalDeps(externalDeps mapset.Set) error {
+	p.Mu.Lock()
+	defer p.Mu.Unlock()
+	p.TransitiveDeps = make([]lockfile.Package, 0, externalDeps.Cardinality())
+	for _, dependency := range externalDeps.ToSlice() {
+		dependency := dependency.(lockfile.Package)
+		p.TransitiveDeps = append(p.TransitiveDeps, dependency)
+	}
+	sort.Sort(lockfile.ByKey(p.TransitiveDeps))
+	hashOfExternalDeps, err := HashObject(p.TransitiveDeps)
+	if err != nil {
+		return err
+	}
+	p.ExternalDepsHash = hashOfExternalDeps
+	return nil
+}
+
 func isEmpty(value interface{}) bool {
 	if value == nil {
 		return true

cli/internal/lockfile/lockfile.go

@@ -61,17 +61,40 @@ func (p ByKey) Less(i, j int) bool {
 
 var _ (sort.Interface) = (*ByKey)(nil)
 
-// TransitiveClosure the set of all lockfile keys that pkg depends on
-func TransitiveClosure(
-	workspaceDir turbopath.AnchoredUnixPath,
-	unresolvedDeps map[string]string,
+type closureMsg struct {
+	workspace turbopath.AnchoredUnixPath
+	closure   mapset.Set
+}
+
+// AllTransitiveClosures computes closures for all workspaces
+func AllTransitiveClosures(
+	workspaces map[turbopath.AnchoredUnixPath]map[string]string,
 	lockFile Lockfile,
-) (mapset.Set, error) {
-	if lf, ok := lockFile.(*NpmLockfile); ok {
-		// We special case as Rust implementations have their own dep crawl
-		return npmTransitiveDeps(lf, workspaceDir, unresolvedDeps)
+) (map[turbopath.AnchoredUnixPath]mapset.Set, error) {
+	g := new(errgroup.Group)
+	c := make(chan closureMsg, len(workspaces))
+	closures := make(map[turbopath.AnchoredUnixPath]mapset.Set, len(workspaces))
+	for workspace, deps := range workspaces {
+		workspace := workspace
+		deps := deps
+		g.Go(func() error {
+			closure, err := transitiveClosure(workspace, deps, lockFile)
+			if err != nil {
+				return err
+			}
+			c <- closureMsg{workspace: workspace, closure: closure}
+			return nil
+		})
+	}
+	err := g.Wait()
+	close(c)
+	if err != nil {
+		return nil, err
+	}
+	for msg := range c {
+		closures[msg.workspace] = msg.closure
 	}
-	return transitiveClosure(workspaceDir, unresolvedDeps, lockFile)
+	return closures, nil
 }
 
 func transitiveClosure(
@@ -83,6 +106,11 @@ func transitiveClosure(
 		return nil, fmt.Errorf("No lockfile available to do analysis on")
 	}
 
+	if lf, ok := lockFile.(*NpmLockfile); ok {
+		// We special case as Rust implementations have their own dep crawl
+		return npmTransitiveDeps(lf, workspaceDir, unresolvedDeps)
+	}
+
 	resolvedPkgs := mapset.NewSet()
 	lockfileEg := &errgroup.Group{}