Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

chore: move lockfile ffi boundary #4629

Merged
merged 18 commits into from Apr 26, 2023
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
43 changes: 24 additions & 19 deletions cli/internal/context/context.go
Expand Up @@ -230,21 +230,17 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f
rootPackageJSON.ExternalDepsHash = ""
} else {
c.Lockfile = lockFile
for _, pkg := range c.WorkspaceInfos.PackageJSONs {
depSet, err := lockfile.TransitiveClosure(
pkg.Dir.ToUnixPath(),
pkg.UnresolvedExternalDeps,
c.Lockfile,
)
if err != nil {
warnings.append(err)
continue
}
if err := pkg.SetExternalDeps(depSet); err != nil {
return nil, err
if closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), c.Lockfile); err != nil {
nathanhammond marked this conversation as resolved.
Show resolved Hide resolved
warnings.append(err)
} else {
for _, pkg := range c.WorkspaceInfos.PackageJSONs {
if closure, ok := closures[pkg.Dir.ToUnixPath()]; ok {
if err := pkg.SetExternalDeps(closure); err != nil {
return nil, err
}
}
}
}

}

return c, warnings.errorOrNil()
Expand Down Expand Up @@ -354,6 +350,14 @@ func (c *Context) parsePackageJSON(repoRoot turbopath.AbsoluteSystemPath, pkgJSO
return nil
}

func (c *Context) externalWorkspaceDeps() map[turbopath.AnchoredUnixPath]map[string]string {
workspaces := make(map[turbopath.AnchoredUnixPath]map[string]string, len(c.WorkspaceInfos.PackageJSONs))
for _, pkg := range c.WorkspaceInfos.PackageJSONs {
workspaces[pkg.Dir.ToUnixPath()] = pkg.UnresolvedExternalDeps
}
return workspaces
}

// InternalDependencies finds all dependencies required by the slice of starting
// packages, as well as the starting packages themselves.
func (c *Context) InternalDependencies(start []string) ([]string, error) {
Expand Down Expand Up @@ -391,13 +395,14 @@ func (c *Context) ChangedPackages(previousLockfile lockfile.Lockfile) ([]string,
return nil, fmt.Errorf("Cannot detect changed packages without previous and current lockfile")
}

closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), previousLockfile)
if err != nil {
return nil, err
}

didPackageChange := func(pkgName string, pkg *fs.PackageJSON) bool {
previousDeps, err := lockfile.TransitiveClosure(
pkg.Dir.ToUnixPath(),
pkg.UnresolvedExternalDeps,
previousLockfile,
)
if err != nil || previousDeps.Cardinality() != len(pkg.TransitiveDeps) {
previousDeps, ok := closures[pkg.Dir.ToUnixPath()]
if !ok || previousDeps.Cardinality() != len(pkg.TransitiveDeps) {
return true
}

Expand Down
46 changes: 37 additions & 9 deletions cli/internal/lockfile/lockfile.go
Expand Up @@ -61,17 +61,40 @@ func (p ByKey) Less(i, j int) bool {

var _ (sort.Interface) = (*ByKey)(nil)

// TransitiveClosure the set of all lockfile keys that pkg depends on
func TransitiveClosure(
workspaceDir turbopath.AnchoredUnixPath,
unresolvedDeps map[string]string,
type closureMsg struct {
workspace turbopath.AnchoredUnixPath
closure mapset.Set
}

// AllTransitiveClosures computes closures for all workspaces
func AllTransitiveClosures(
workspaces map[turbopath.AnchoredUnixPath]map[string]string,
lockFile Lockfile,
) (mapset.Set, error) {
if lf, ok := lockFile.(*NpmLockfile); ok {
// We special case as Rust implementations have their own dep crawl
return npmTransitiveDeps(lf, workspaceDir, unresolvedDeps)
) (map[turbopath.AnchoredUnixPath]mapset.Set, error) {
g := new(errgroup.Group)
c := make(chan closureMsg, len(workspaces))
closures := make(map[turbopath.AnchoredUnixPath]mapset.Set, len(workspaces))
for workspace, deps := range workspaces {
workspace := workspace
deps := deps
g.Go(func() error {
closure, err := transitiveClosure(workspace, deps, lockFile)
if err != nil {
return err
}
c <- closureMsg{workspace: workspace, closure: closure}
return nil
})
}
err := g.Wait()
close(c)
if err != nil {
return nil, err
}
for msg := range c {
closures[msg.workspace] = msg.closure
}
return transitiveClosure(workspaceDir, unresolvedDeps, lockFile)
return closures, nil
}

func transitiveClosure(
Expand All @@ -83,6 +106,11 @@ func transitiveClosure(
return nil, fmt.Errorf("No lockfile available to do analysis on")
}

if lf, ok := lockFile.(*NpmLockfile); ok {
// We special case as Rust implementations have their own dep crawl
return npmTransitiveDeps(lf, workspaceDir, unresolvedDeps)
}

resolvedPkgs := mapset.NewSet()
lockfileEg := &errgroup.Group{}

Expand Down