New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
chore: move lockfile ffi boundary #4629
Changes from 6 commits
8c99b57
8d93b1e
ab9f0b3
f9a460c
4f9c395
31ef289
1c6d663
92f0b58
2efd867
57be317
0ff9dbc
1931953
fbdeffd
8c6c647
b0f20e7
d1c52da
fada352
9049e5c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -17,7 +17,6 @@ import ( | |
"github.com/vercel/turbo/cli/internal/workspace" | ||
|
||
"github.com/Masterminds/semver" | ||
mapset "github.com/deckarep/golang-set" | ||
"github.com/pyr-sh/dag" | ||
"golang.org/x/sync/errgroup" | ||
) | ||
|
@@ -179,12 +178,6 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f | |
} | ||
c.PackageManager = packageManager | ||
|
||
if lockfile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil { | ||
warnings.append(err) | ||
} else { | ||
c.Lockfile = lockfile | ||
} | ||
|
||
if err := c.resolveWorkspaceRootDeps(rootPackageJSON, &warnings); err != nil { | ||
// TODO(Gaspar) was this the intended return error? | ||
return nil, fmt.Errorf("could not resolve workspaces: %w", err) | ||
|
@@ -232,6 +225,25 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f | |
} | ||
c.WorkspaceInfos.PackageJSONs[util.RootPkgName] = rootPackageJSON | ||
|
||
if lockFile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil { | ||
warnings.append(err) | ||
rootPackageJSON.TransitiveDeps = []lockfile.Package{} | ||
rootPackageJSON.ExternalDepsHash = "" | ||
} else { | ||
c.Lockfile = lockFile | ||
if closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), c.Lockfile); err != nil { | ||
nathanhammond marked this conversation as resolved.
Show resolved
Hide resolved
|
||
warnings.append(err) | ||
} else { | ||
for _, pkg := range c.WorkspaceInfos.PackageJSONs { | ||
if closure, ok := closures[pkg.Dir.ToUnixPath()]; ok { | ||
if err := pkg.SetExternalDeps(closure); err != nil { | ||
return nil, err | ||
} | ||
} | ||
} | ||
} | ||
} | ||
|
||
return c, warnings.errorOrNil() | ||
} | ||
|
||
|
@@ -247,33 +259,6 @@ func (c *Context) resolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON, warn | |
for dep, version := range pkg.Dependencies { | ||
pkg.UnresolvedExternalDeps[dep] = version | ||
} | ||
if c.Lockfile != nil { | ||
depSet, err := lockfile.TransitiveClosure( | ||
pkg.Dir.ToUnixPath(), | ||
pkg.UnresolvedExternalDeps, | ||
c.Lockfile, | ||
) | ||
if err != nil { | ||
warnings.append(err) | ||
// Return early to skip using results of incomplete dep graph resolution | ||
return nil | ||
} | ||
pkg.TransitiveDeps = make([]lockfile.Package, 0, depSet.Cardinality()) | ||
for _, v := range depSet.ToSlice() { | ||
dep := v.(lockfile.Package) | ||
pkg.TransitiveDeps = append(pkg.TransitiveDeps, dep) | ||
} | ||
sort.Sort(lockfile.ByKey(pkg.TransitiveDeps)) | ||
hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps) | ||
if err != nil { | ||
return err | ||
} | ||
pkg.ExternalDepsHash = hashOfExternalDeps | ||
} else { | ||
pkg.TransitiveDeps = []lockfile.Package{} | ||
pkg.ExternalDepsHash = "" | ||
} | ||
|
||
return nil | ||
} | ||
|
||
|
@@ -326,37 +311,18 @@ func (c *Context) populateWorkspaceGraphForPackageJSON(pkg *fs.PackageJSON, root | |
} | ||
} | ||
|
||
externalDeps, err := lockfile.TransitiveClosure( | ||
pkg.Dir.ToUnixPath(), | ||
pkg.UnresolvedExternalDeps, | ||
c.Lockfile, | ||
) | ||
if err != nil { | ||
warnings.append(err) | ||
// reset external deps to original state | ||
externalDeps = mapset.NewSet() | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. What was this doing? There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This was here instead of proper control flow, if the closure calculation errored we would override the return value with an empty set. This would then get translated to |
||
} | ||
|
||
// when there are no internal dependencies, we need to still add these leafs to the graph | ||
if internalDepsSet.Len() == 0 { | ||
c.WorkspaceGraph.Connect(dag.BasicEdge(pkg.Name, core.ROOT_NODE_NAME)) | ||
} | ||
pkg.TransitiveDeps = make([]lockfile.Package, 0, externalDeps.Cardinality()) | ||
for _, dependency := range externalDeps.ToSlice() { | ||
dependency := dependency.(lockfile.Package) | ||
pkg.TransitiveDeps = append(pkg.TransitiveDeps, dependency) | ||
} | ||
|
||
pkg.InternalDeps = make([]string, 0, internalDepsSet.Len()) | ||
for _, v := range internalDepsSet.List() { | ||
pkg.InternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%v", v)) | ||
} | ||
|
||
sort.Strings(pkg.InternalDeps) | ||
sort.Sort(lockfile.ByKey(pkg.TransitiveDeps)) | ||
hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps) | ||
if err != nil { | ||
return err | ||
} | ||
pkg.ExternalDepsHash = hashOfExternalDeps | ||
|
||
return nil | ||
} | ||
|
||
|
@@ -387,6 +353,14 @@ func (c *Context) parsePackageJSON(repoRoot turbopath.AbsoluteSystemPath, pkgJSO | |
return nil | ||
} | ||
|
||
func (c *Context) externalWorkspaceDeps() map[turbopath.AnchoredUnixPath]map[string]string { | ||
workspaces := make(map[turbopath.AnchoredUnixPath]map[string]string, len(c.WorkspaceInfos.PackageJSONs)) | ||
for _, pkg := range c.WorkspaceInfos.PackageJSONs { | ||
workspaces[pkg.Dir.ToUnixPath()] = pkg.UnresolvedExternalDeps | ||
} | ||
return workspaces | ||
} | ||
|
||
// InternalDependencies finds all dependencies required by the slice of starting | ||
// packages, as well as the starting packages themselves. | ||
func (c *Context) InternalDependencies(start []string) ([]string, error) { | ||
|
@@ -424,13 +398,14 @@ func (c *Context) ChangedPackages(previousLockfile lockfile.Lockfile) ([]string, | |
return nil, fmt.Errorf("Cannot detect changed packages without previous and current lockfile") | ||
} | ||
|
||
closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), previousLockfile) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
didPackageChange := func(pkgName string, pkg *fs.PackageJSON) bool { | ||
previousDeps, err := lockfile.TransitiveClosure( | ||
pkg.Dir.ToUnixPath(), | ||
pkg.UnresolvedExternalDeps, | ||
previousLockfile, | ||
) | ||
if err != nil || previousDeps.Cardinality() != len(pkg.TransitiveDeps) { | ||
previousDeps, ok := closures[pkg.Dir.ToUnixPath()] | ||
if !ok || previousDeps.Cardinality() != len(pkg.TransitiveDeps) { | ||
return true | ||
} | ||
|
||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -168,23 +168,28 @@ func PreviousContent(repoRoot, fromCommit, filePath string) ([]byte, error) { | |
return []byte(content), nil | ||
} | ||
|
||
// NpmTransitiveDeps returns the transitive external deps of a given package based on the deps and specifiers given | ||
func NpmTransitiveDeps(content []byte, pkgDir string, unresolvedDeps map[string]string) ([]*ffi_proto.LockfilePackage, error) { | ||
return transitiveDeps(npmTransitiveDeps, content, pkgDir, unresolvedDeps) | ||
} | ||
|
||
func npmTransitiveDeps(buf C.Buffer) C.Buffer { | ||
return C.npm_transitive_closure(buf) | ||
} | ||
|
||
func transitiveDeps(cFunc func(C.Buffer) C.Buffer, content []byte, pkgDir string, unresolvedDeps map[string]string) ([]*ffi_proto.LockfilePackage, error) { | ||
// TransitiveDeps returns the transitive external deps for all provided workspaces | ||
func TransitiveDeps(content []byte, packageManager string, workspaces map[string]map[string]string) (map[string]*ffi_proto.LockfilePackageList, error) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Confirming understanding: this has to do a full walk because if there are floating versions the top-level deps could have not changed, but somewhere way down the tree could have. There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Exactly, if someone blows away the lockfile and does a fresh install all of the top level specifiers will remain the same, but what version gets installed might change. |
||
flatWorkspaces := make(map[string]*ffi_proto.PackageDependencyList) | ||
for workspace, deps := range workspaces { | ||
packageDependencyList := make([]*ffi_proto.PackageDependency, len(deps)) | ||
i := 0 | ||
for name, version := range deps { | ||
packageDependencyList[i] = &ffi_proto.PackageDependency{ | ||
Name: name, | ||
Range: version, | ||
} | ||
i++ | ||
} | ||
flatWorkspaces[workspace] = &ffi_proto.PackageDependencyList{List: packageDependencyList} | ||
} | ||
req := ffi_proto.TransitiveDepsRequest{ | ||
Contents: content, | ||
WorkspaceDir: pkgDir, | ||
UnresolvedDeps: unresolvedDeps, | ||
PackageManager: packageManager, | ||
Workspaces: flatWorkspaces, | ||
} | ||
reqBuf := Marshal(&req) | ||
resBuf := cFunc(reqBuf) | ||
resBuf := C.transitive_closure(reqBuf) | ||
reqBuf.Free() | ||
|
||
resp := ffi_proto.TransitiveDepsResponse{} | ||
|
@@ -196,8 +201,8 @@ func transitiveDeps(cFunc func(C.Buffer) C.Buffer, content []byte, pkgDir string | |
return nil, errors.New(err) | ||
} | ||
|
||
list := resp.GetPackages() | ||
return list.GetList(), nil | ||
dependencies := resp.GetDependencies() | ||
return dependencies.GetDependencies(), nil | ||
} | ||
|
||
// NpmSubgraph returns the contents of a npm lockfile subgraph | ||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Does this work as just
nil
?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Haven't done a full code trace, but it should. This was just copied from previous behavior.
Edit: Checked and
nil
works just fine here