chore: move lockfile ffi boundary

cli/internal/context/context.go

@@ -17,7 +17,6 @@ import (
 	"github.com/vercel/turbo/cli/internal/workspace"
 
 	"github.com/Masterminds/semver"
-	mapset "github.com/deckarep/golang-set"
 	"github.com/pyr-sh/dag"
 	"golang.org/x/sync/errgroup"
 )
@@ -179,12 +178,6 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
 	}
 	c.PackageManager = packageManager
 
-	if lockfile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil {
-		warnings.append(err)
-	} else {
-		c.Lockfile = lockfile
-	}
-
 	if err := c.resolveWorkspaceRootDeps(rootPackageJSON, &warnings); err != nil {
 		// TODO(Gaspar) was this the intended return error?
 		return nil, fmt.Errorf("could not resolve workspaces: %w", err)
@@ -214,10 +207,9 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
 	}
 	populateGraphWaitGroup := &errgroup.Group{}
 	for _, pkg := range c.WorkspaceInfos.PackageJSONs {
-		pkg := pkg
-		populateGraphWaitGroup.Go(func() error {
-			return c.populateWorkspaceGraphForPackageJSON(pkg, rootpath, pkg.Name, &warnings)
-		})
+		if err := c.populateWorkspaceGraphForPackageJSON(pkg, rootpath, pkg.Name, &warnings); err != nil {
+			return nil, err
+		}
 	}
 
 	if err := populateGraphWaitGroup.Wait(); err != nil {
@@ -232,6 +224,25 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
 	}
 	c.WorkspaceInfos.PackageJSONs[util.RootPkgName] = rootPackageJSON
 
+	if lockFile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil {
+		warnings.append(err)
+		rootPackageJSON.TransitiveDeps = []lockfile.Package{}
+		rootPackageJSON.ExternalDepsHash = ""
+	} else {
+		c.Lockfile = lockFile
+		if closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), c.Lockfile); err != nil {
+			warnings.append(err)
+		} else {
+			for _, pkg := range c.WorkspaceInfos.PackageJSONs {
+				if closure, ok := closures[pkg.Dir.ToUnixPath()]; ok {
+					if err := pkg.SetExternalDeps(closure); err != nil {
+						return nil, err
+					}
+				}
+			}
+		}
+	}
+
 	return c, warnings.errorOrNil()
 }
 
@@ -247,33 +258,6 @@ func (c *Context) resolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON, warn
 	for dep, version := range pkg.Dependencies {
 		pkg.UnresolvedExternalDeps[dep] = version
 	}
-	if c.Lockfile != nil {
-		depSet, err := lockfile.TransitiveClosure(
-			pkg.Dir.ToUnixPath(),
-			pkg.UnresolvedExternalDeps,
-			c.Lockfile,
-		)
-		if err != nil {
-			warnings.append(err)
-			// Return early to skip using results of incomplete dep graph resolution
-			return nil
-		}
-		pkg.TransitiveDeps = make([]lockfile.Package, 0, depSet.Cardinality())
-		for _, v := range depSet.ToSlice() {
-			dep := v.(lockfile.Package)
-			pkg.TransitiveDeps = append(pkg.TransitiveDeps, dep)
-		}
-		sort.Sort(lockfile.ByKey(pkg.TransitiveDeps))
-		hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps)
-		if err != nil {
-			return err
-		}
-		pkg.ExternalDepsHash = hashOfExternalDeps
-	} else {
-		pkg.TransitiveDeps = []lockfile.Package{}
-		pkg.ExternalDepsHash = ""
-	}
-
 	return nil
 }
 
@@ -282,8 +266,6 @@ func (c *Context) resolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON, warn
 // that are not within the monorepo. The vertexName is used to override the package name in the graph.
 // This can happen when adding the root package, which can have an arbitrary name.
 func (c *Context) populateWorkspaceGraphForPackageJSON(pkg *fs.PackageJSON, rootpath string, vertexName string, warnings *Warnings) error {
-	c.mutex.Lock()
-	defer c.mutex.Unlock()
 	depMap := make(map[string]string)
 	internalDepsSet := make(dag.Set)
 	externalUnresolvedDepsSet := make(dag.Set)
@@ -326,37 +308,18 @@ func (c *Context) populateWorkspaceGraphForPackageJSON(pkg *fs.PackageJSON, root
 		}
 	}
 
-	externalDeps, err := lockfile.TransitiveClosure(
-		pkg.Dir.ToUnixPath(),
-		pkg.UnresolvedExternalDeps,
-		c.Lockfile,
-	)
-	if err != nil {
-		warnings.append(err)
-		// reset external deps to original state
-		externalDeps = mapset.NewSet()
-	}
-
 	// when there are no internal dependencies, we need to still add these leafs to the graph
 	if internalDepsSet.Len() == 0 {
 		c.WorkspaceGraph.Connect(dag.BasicEdge(pkg.Name, core.ROOT_NODE_NAME))
 	}
-	pkg.TransitiveDeps = make([]lockfile.Package, 0, externalDeps.Cardinality())
-	for _, dependency := range externalDeps.ToSlice() {
-		dependency := dependency.(lockfile.Package)
-		pkg.TransitiveDeps = append(pkg.TransitiveDeps, dependency)
-	}
+
 	pkg.InternalDeps = make([]string, 0, internalDepsSet.Len())
 	for _, v := range internalDepsSet.List() {
 		pkg.InternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%v", v))
 	}
+
 	sort.Strings(pkg.InternalDeps)
-	sort.Sort(lockfile.ByKey(pkg.TransitiveDeps))
-	hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps)
-	if err != nil {
-		return err
-	}
-	pkg.ExternalDepsHash = hashOfExternalDeps
+
 	return nil
 }
 
@@ -387,6 +350,14 @@ func (c *Context) parsePackageJSON(repoRoot turbopath.AbsoluteSystemPath, pkgJSO
 	return nil
 }
 
+func (c *Context) externalWorkspaceDeps() map[turbopath.AnchoredUnixPath]map[string]string {
+	workspaces := make(map[turbopath.AnchoredUnixPath]map[string]string, len(c.WorkspaceInfos.PackageJSONs))
+	for _, pkg := range c.WorkspaceInfos.PackageJSONs {
+		workspaces[pkg.Dir.ToUnixPath()] = pkg.UnresolvedExternalDeps
+	}
+	return workspaces
+}
+
 // InternalDependencies finds all dependencies required by the slice of starting
 // packages, as well as the starting packages themselves.
 func (c *Context) InternalDependencies(start []string) ([]string, error) {
@@ -424,13 +395,14 @@ func (c *Context) ChangedPackages(previousLockfile lockfile.Lockfile) ([]string,
 		return nil, fmt.Errorf("Cannot detect changed packages without previous and current lockfile")
 	}
 
+	closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), previousLockfile)
+	if err != nil {
+		return nil, err
+	}
+
 	didPackageChange := func(pkgName string, pkg *fs.PackageJSON) bool {
-		previousDeps, err := lockfile.TransitiveClosure(
-			pkg.Dir.ToUnixPath(),
-			pkg.UnresolvedExternalDeps,
-			previousLockfile,
-		)
-		if err != nil || previousDeps.Cardinality() != len(pkg.TransitiveDeps) {
+		previousDeps, ok := closures[pkg.Dir.ToUnixPath()]
+		if !ok || previousDeps.Cardinality() != len(pkg.TransitiveDeps) {
 			return true
 		}
 

cli/internal/ffi/bindings.h

@@ -16,6 +16,6 @@ struct Buffer changed_files(struct Buffer buffer);
 
 struct Buffer previous_content(struct Buffer buffer);
 
-struct Buffer npm_transitive_closure(struct Buffer buf);
+struct Buffer transitive_closure(struct Buffer buf);
 
 struct Buffer npm_subgraph(struct Buffer buf);

cli/internal/ffi/ffi.go

@@ -168,23 +168,28 @@ func PreviousContent(repoRoot, fromCommit, filePath string) ([]byte, error) {
 	return []byte(content), nil
 }
 
-// NpmTransitiveDeps returns the transitive external deps of a given package based on the deps and specifiers given
-func NpmTransitiveDeps(content []byte, pkgDir string, unresolvedDeps map[string]string) ([]*ffi_proto.LockfilePackage, error) {
-	return transitiveDeps(npmTransitiveDeps, content, pkgDir, unresolvedDeps)
-}
-
-func npmTransitiveDeps(buf C.Buffer) C.Buffer {
-	return C.npm_transitive_closure(buf)
-}
-
-func transitiveDeps(cFunc func(C.Buffer) C.Buffer, content []byte, pkgDir string, unresolvedDeps map[string]string) ([]*ffi_proto.LockfilePackage, error) {
+// TransitiveDeps returns the transitive external deps for all provided workspaces
+func TransitiveDeps(content []byte, packageManager string, workspaces map[string]map[string]string) (map[string]*ffi_proto.LockfilePackageList, error) {
+	flatWorkspaces := make(map[string]*ffi_proto.PackageDependencyList)
+	for workspace, deps := range workspaces {
+		packageDependencyList := make([]*ffi_proto.PackageDependency, len(deps))
+		i := 0
+		for name, version := range deps {
+			packageDependencyList[i] = &ffi_proto.PackageDependency{
+				Name:  name,
+				Range: version,
+			}
+			i++
+		}
+		flatWorkspaces[workspace] = &ffi_proto.PackageDependencyList{List: packageDependencyList}
+	}
 	req := ffi_proto.TransitiveDepsRequest{
 		Contents:       content,
-		WorkspaceDir:   pkgDir,
-		UnresolvedDeps: unresolvedDeps,
+		PackageManager: packageManager,
+		Workspaces:     flatWorkspaces,
 	}
 	reqBuf := Marshal(&req)
-	resBuf := cFunc(reqBuf)
+	resBuf := C.transitive_closure(reqBuf)
 	reqBuf.Free()
 
 	resp := ffi_proto.TransitiveDepsResponse{}
@@ -196,8 +201,8 @@ func transitiveDeps(cFunc func(C.Buffer) C.Buffer, content []byte, pkgDir string
 		return nil, errors.New(err)
 	}
 
-	list := resp.GetPackages()
-	return list.GetList(), nil
+	dependencies := resp.GetDependencies()
+	return dependencies.GetDependencies(), nil
 }
 
 // NpmSubgraph returns the contents of a npm lockfile subgraph