From 1c8d3aa9621f171a177db0eb068f3027a756ce1a Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Wed, 26 Apr 2023 11:21:34 -0700 Subject: [PATCH] chore: move lockfile ffi boundary (#4629) ### Description In order to avoid Go holding onto memory that's been allocated by Rust we parse the lockfile each time we need to make a lockfile call. This worked fine for the npm implementation as we just needed parse the JSON and it was usable, but Berry requires a lot more work/allocations that makes this strategy unfeasible. A quick sketch of the changes in this PR: - Moves the traversal of the lockfile to be the final step of building the package graph - We now calculate all of the closures of all workspaces at once - Rust FFI now takes the package manager string to select the lockfile implementation. This is a quick prefactor to prepare for hooking up the berry lockfile Reviewer notes: - Each commit can be reviewed on it's own, the first commit ended up getting reverted down the stack so it can be skipped. ### Testing Instructions Existing unit tests/integration tests. Manual verification of correct pruning behavior for the npm and pnpm monorepos created by `create-turbo@latest` --- cli/internal/context/context.go | 109 ++--- cli/internal/context/context_test.go | 37 ++ cli/internal/ffi/bindings.h | 2 +- cli/internal/ffi/ffi.go | 45 +- cli/internal/ffi/proto/messages.pb.go | 511 +++++++++++++++----- cli/internal/fs/package_json.go | 20 + cli/internal/fs/package_json_test.go | 10 + cli/internal/lockfile/lockfile.go | 68 ++- cli/internal/lockfile/lockfile_test.go | 45 ++ cli/internal/lockfile/npm_lockfile.go | 21 - cli/internal/lockfile/npm_lockfile_test.go | 74 +++ crates/turborepo-ffi/messages.proto | 28 +- crates/turborepo-ffi/src/lib.rs | 2 +- crates/turborepo-ffi/src/lockfile.rs | 83 +++- crates/turborepo-lockfiles/src/berry/mod.rs | 3 +- crates/turborepo-lockfiles/src/lib.rs | 4 +- 16 files changed, 811 insertions(+), 251 deletions(-) create mode 100644 cli/internal/lockfile/npm_lockfile_test.go diff --git a/cli/internal/context/context.go b/cli/internal/context/context.go index 2376d2d9eb32a..7e556a758facc 100644 --- a/cli/internal/context/context.go +++ b/cli/internal/context/context.go @@ -17,7 +17,6 @@ import ( "github.com/vercel/turbo/cli/internal/workspace" "github.com/Masterminds/semver" - mapset "github.com/deckarep/golang-set" "github.com/pyr-sh/dag" "golang.org/x/sync/errgroup" ) @@ -179,12 +178,6 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f } c.PackageManager = packageManager - if lockfile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil { - warnings.append(err) - } else { - c.Lockfile = lockfile - } - if err := c.resolveWorkspaceRootDeps(rootPackageJSON, &warnings); err != nil { // TODO(Gaspar) was this the intended return error? return nil, fmt.Errorf("could not resolve workspaces: %w", err) @@ -232,6 +225,10 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f } c.WorkspaceInfos.PackageJSONs[util.RootPkgName] = rootPackageJSON + if err := c.populateExternalDeps(repoRoot, rootPackageJSON, &warnings); err != nil { + return nil, err + } + return c, warnings.errorOrNil() } @@ -247,33 +244,6 @@ func (c *Context) resolveWorkspaceRootDeps(rootPackageJSON *fs.PackageJSON, warn for dep, version := range pkg.Dependencies { pkg.UnresolvedExternalDeps[dep] = version } - if c.Lockfile != nil { - depSet, err := lockfile.TransitiveClosure( - pkg.Dir.ToUnixPath(), - pkg.UnresolvedExternalDeps, - c.Lockfile, - ) - if err != nil { - warnings.append(err) - // Return early to skip using results of incomplete dep graph resolution - return nil - } - pkg.TransitiveDeps = make([]lockfile.Package, 0, depSet.Cardinality()) - for _, v := range depSet.ToSlice() { - dep := v.(lockfile.Package) - pkg.TransitiveDeps = append(pkg.TransitiveDeps, dep) - } - sort.Sort(lockfile.ByKey(pkg.TransitiveDeps)) - hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps) - if err != nil { - return err - } - pkg.ExternalDepsHash = hashOfExternalDeps - } else { - pkg.TransitiveDeps = []lockfile.Package{} - pkg.ExternalDepsHash = "" - } - return nil } @@ -326,37 +296,18 @@ func (c *Context) populateWorkspaceGraphForPackageJSON(pkg *fs.PackageJSON, root } } - externalDeps, err := lockfile.TransitiveClosure( - pkg.Dir.ToUnixPath(), - pkg.UnresolvedExternalDeps, - c.Lockfile, - ) - if err != nil { - warnings.append(err) - // reset external deps to original state - externalDeps = mapset.NewSet() - } - // when there are no internal dependencies, we need to still add these leafs to the graph if internalDepsSet.Len() == 0 { c.WorkspaceGraph.Connect(dag.BasicEdge(pkg.Name, core.ROOT_NODE_NAME)) } - pkg.TransitiveDeps = make([]lockfile.Package, 0, externalDeps.Cardinality()) - for _, dependency := range externalDeps.ToSlice() { - dependency := dependency.(lockfile.Package) - pkg.TransitiveDeps = append(pkg.TransitiveDeps, dependency) - } + pkg.InternalDeps = make([]string, 0, internalDepsSet.Len()) for _, v := range internalDepsSet.List() { pkg.InternalDeps = append(pkg.InternalDeps, fmt.Sprintf("%v", v)) } + sort.Strings(pkg.InternalDeps) - sort.Sort(lockfile.ByKey(pkg.TransitiveDeps)) - hashOfExternalDeps, err := fs.HashObject(pkg.TransitiveDeps) - if err != nil { - return err - } - pkg.ExternalDepsHash = hashOfExternalDeps + return nil } @@ -387,6 +338,39 @@ func (c *Context) parsePackageJSON(repoRoot turbopath.AbsoluteSystemPath, pkgJSO return nil } +func (c *Context) externalWorkspaceDeps() map[turbopath.AnchoredUnixPath]map[string]string { + workspaces := make(map[turbopath.AnchoredUnixPath]map[string]string, len(c.WorkspaceInfos.PackageJSONs)) + for _, pkg := range c.WorkspaceInfos.PackageJSONs { + workspaces[pkg.Dir.ToUnixPath()] = pkg.UnresolvedExternalDeps + } + return workspaces +} + +func (c *Context) populateExternalDeps(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *fs.PackageJSON, warnings *Warnings) error { + if lockFile, err := c.PackageManager.ReadLockfile(repoRoot, rootPackageJSON); err != nil { + warnings.append(err) + rootPackageJSON.TransitiveDeps = nil + rootPackageJSON.ExternalDepsHash = "" + } else { + c.Lockfile = lockFile + if closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), c.Lockfile); err != nil { + warnings.append(err) + } else { + for _, pkg := range c.WorkspaceInfos.PackageJSONs { + if closure, ok := closures[pkg.Dir.ToUnixPath()]; ok { + if err := pkg.SetExternalDeps(closure); err != nil { + return err + } + } else { + return fmt.Errorf("Unable to calculate closure for workspace %s", pkg.Dir.ToString()) + } + } + } + } + + return nil +} + // InternalDependencies finds all dependencies required by the slice of starting // packages, as well as the starting packages themselves. func (c *Context) InternalDependencies(start []string) ([]string, error) { @@ -424,13 +408,14 @@ func (c *Context) ChangedPackages(previousLockfile lockfile.Lockfile) ([]string, return nil, fmt.Errorf("Cannot detect changed packages without previous and current lockfile") } + closures, err := lockfile.AllTransitiveClosures(c.externalWorkspaceDeps(), previousLockfile) + if err != nil { + return nil, err + } + didPackageChange := func(pkgName string, pkg *fs.PackageJSON) bool { - previousDeps, err := lockfile.TransitiveClosure( - pkg.Dir.ToUnixPath(), - pkg.UnresolvedExternalDeps, - previousLockfile, - ) - if err != nil || previousDeps.Cardinality() != len(pkg.TransitiveDeps) { + previousDeps, ok := closures[pkg.Dir.ToUnixPath()] + if !ok || previousDeps.Cardinality() != len(pkg.TransitiveDeps) { return true } diff --git a/cli/internal/context/context_test.go b/cli/internal/context/context_test.go index 692c0a8b694a5..7e04590416c25 100644 --- a/cli/internal/context/context_test.go +++ b/cli/internal/context/context_test.go @@ -1,14 +1,20 @@ package context import ( + "errors" "os" "path/filepath" "regexp" + "sync" "testing" testifyAssert "github.com/stretchr/testify/assert" "github.com/vercel/turbo/cli/internal/fs" + "github.com/vercel/turbo/cli/internal/lockfile" + "github.com/vercel/turbo/cli/internal/packagemanager" "github.com/vercel/turbo/cli/internal/turbopath" + "github.com/vercel/turbo/cli/internal/workspace" + "gotest.tools/v3/assert" ) func Test_isWorkspaceReference(t *testing.T) { @@ -144,6 +150,37 @@ func TestBuildPackageGraph_DuplicateNames(t *testing.T) { testifyAssert.Regexp(t, regexp.MustCompile("^Failed to add workspace \"same-name\".+$"), actualErr) } +func Test_populateExternalDeps_NoTransitiveDepsWithoutLockfile(t *testing.T) { + path := getTestDir(t, "dupe-workspace-names") + pkgJSON := &fs.PackageJSON{ + Name: "dupe-workspace-names", + PackageManager: "pnpm@7.15.0", + } + + pm, err := packagemanager.GetPackageManager(path, pkgJSON) + assert.NilError(t, err) + pm.UnmarshalLockfile = func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) { + return nil, errors.New("bad lockfile") + } + context := Context{ + WorkspaceInfos: workspace.Catalog{ + PackageJSONs: map[string]*fs.PackageJSON{ + "a": {}, + }, + }, + WorkspaceNames: []string{}, + PackageManager: pm, + mutex: sync.Mutex{}, + } + var warnings Warnings + err = context.populateExternalDeps(path, pkgJSON, &warnings) + assert.NilError(t, err) + + assert.DeepEqual(t, pkgJSON.ExternalDepsHash, "") + assert.DeepEqual(t, context.WorkspaceInfos.PackageJSONs["a"].ExternalDepsHash, "") + assert.Assert(t, warnings.errorOrNil() != nil) +} + // This is duplicated from fs.turbo_json_test.go. // I wasn't able to pull it into a helper file/package because // it requires the `fs` package and it would cause cyclical dependencies diff --git a/cli/internal/ffi/bindings.h b/cli/internal/ffi/bindings.h index c2bbcea5d5f06..53f9bd0d86f2f 100644 --- a/cli/internal/ffi/bindings.h +++ b/cli/internal/ffi/bindings.h @@ -16,6 +16,6 @@ struct Buffer changed_files(struct Buffer buffer); struct Buffer previous_content(struct Buffer buffer); -struct Buffer npm_transitive_closure(struct Buffer buf); +struct Buffer transitive_closure(struct Buffer buf); struct Buffer npm_subgraph(struct Buffer buf); diff --git a/cli/internal/ffi/ffi.go b/cli/internal/ffi/ffi.go index d767488509414..82646b19a0836 100644 --- a/cli/internal/ffi/ffi.go +++ b/cli/internal/ffi/ffi.go @@ -18,6 +18,7 @@ import "C" import ( "errors" + "fmt" "reflect" "unsafe" @@ -167,23 +168,28 @@ func PreviousContent(gitRoot, fromCommit, filePath string) ([]byte, error) { return []byte(content), nil } -// NpmTransitiveDeps returns the transitive external deps of a given package based on the deps and specifiers given -func NpmTransitiveDeps(content []byte, pkgDir string, unresolvedDeps map[string]string) ([]*ffi_proto.LockfilePackage, error) { - return transitiveDeps(npmTransitiveDeps, content, pkgDir, unresolvedDeps) -} - -func npmTransitiveDeps(buf C.Buffer) C.Buffer { - return C.npm_transitive_closure(buf) -} - -func transitiveDeps(cFunc func(C.Buffer) C.Buffer, content []byte, pkgDir string, unresolvedDeps map[string]string) ([]*ffi_proto.LockfilePackage, error) { +// TransitiveDeps returns the transitive external deps for all provided workspaces +func TransitiveDeps(content []byte, packageManager string, workspaces map[string]map[string]string) (map[string]*ffi_proto.LockfilePackageList, error) { + flatWorkspaces := make(map[string]*ffi_proto.PackageDependencyList) + for workspace, deps := range workspaces { + packageDependencyList := make([]*ffi_proto.PackageDependency, len(deps)) + i := 0 + for name, version := range deps { + packageDependencyList[i] = &ffi_proto.PackageDependency{ + Name: name, + Range: version, + } + i++ + } + flatWorkspaces[workspace] = &ffi_proto.PackageDependencyList{List: packageDependencyList} + } req := ffi_proto.TransitiveDepsRequest{ Contents: content, - WorkspaceDir: pkgDir, - UnresolvedDeps: unresolvedDeps, + PackageManager: toPackageManager(packageManager), + Workspaces: flatWorkspaces, } reqBuf := Marshal(&req) - resBuf := cFunc(reqBuf) + resBuf := C.transitive_closure(reqBuf) reqBuf.Free() resp := ffi_proto.TransitiveDepsResponse{} @@ -195,8 +201,17 @@ func transitiveDeps(cFunc func(C.Buffer) C.Buffer, content []byte, pkgDir string return nil, errors.New(err) } - list := resp.GetPackages() - return list.GetList(), nil + dependencies := resp.GetDependencies() + return dependencies.GetDependencies(), nil +} + +func toPackageManager(packageManager string) ffi_proto.PackageManager { + switch packageManager { + case "npm": + return ffi_proto.PackageManager_NPM + default: + panic(fmt.Sprintf("Invalid package manager string: %s", packageManager)) + } } // NpmSubgraph returns the contents of a npm lockfile subgraph diff --git a/cli/internal/ffi/proto/messages.pb.go b/cli/internal/ffi/proto/messages.pb.go index 666f32a164416..68a155a44eca1 100644 --- a/cli/internal/ffi/proto/messages.pb.go +++ b/cli/internal/ffi/proto/messages.pb.go @@ -20,6 +20,49 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type PackageManager int32 + +const ( + PackageManager_NPM PackageManager = 0 +) + +// Enum value maps for PackageManager. +var ( + PackageManager_name = map[int32]string{ + 0: "NPM", + } + PackageManager_value = map[string]int32{ + "NPM": 0, + } +) + +func (x PackageManager) Enum() *PackageManager { + p := new(PackageManager) + *p = x + return p +} + +func (x PackageManager) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (PackageManager) Descriptor() protoreflect.EnumDescriptor { + return file_turborepo_ffi_messages_proto_enumTypes[0].Descriptor() +} + +func (PackageManager) Type() protoreflect.EnumType { + return &file_turborepo_ffi_messages_proto_enumTypes[0] +} + +func (x PackageManager) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use PackageManager.Descriptor instead. +func (PackageManager) EnumDescriptor() ([]byte, []int) { + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{0} +} + type TurboDataDirResp struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache @@ -606,20 +649,169 @@ func (*PreviousContentResp_Content) isPreviousContentResp_Response() {} func (*PreviousContentResp_Error) isPreviousContentResp_Response() {} +type PackageDependency struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` + Range string `protobuf:"bytes,2,opt,name=range,proto3" json:"range,omitempty"` +} + +func (x *PackageDependency) Reset() { + *x = PackageDependency{} + if protoimpl.UnsafeEnabled { + mi := &file_turborepo_ffi_messages_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PackageDependency) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PackageDependency) ProtoMessage() {} + +func (x *PackageDependency) ProtoReflect() protoreflect.Message { + mi := &file_turborepo_ffi_messages_proto_msgTypes[9] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PackageDependency.ProtoReflect.Descriptor instead. +func (*PackageDependency) Descriptor() ([]byte, []int) { + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{9} +} + +func (x *PackageDependency) GetName() string { + if x != nil { + return x.Name + } + return "" +} + +func (x *PackageDependency) GetRange() string { + if x != nil { + return x.Range + } + return "" +} + +type PackageDependencyList struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + List []*PackageDependency `protobuf:"bytes,1,rep,name=list,proto3" json:"list,omitempty"` +} + +func (x *PackageDependencyList) Reset() { + *x = PackageDependencyList{} + if protoimpl.UnsafeEnabled { + mi := &file_turborepo_ffi_messages_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *PackageDependencyList) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*PackageDependencyList) ProtoMessage() {} + +func (x *PackageDependencyList) ProtoReflect() protoreflect.Message { + mi := &file_turborepo_ffi_messages_proto_msgTypes[10] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use PackageDependencyList.ProtoReflect.Descriptor instead. +func (*PackageDependencyList) Descriptor() ([]byte, []int) { + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{10} +} + +func (x *PackageDependencyList) GetList() []*PackageDependency { + if x != nil { + return x.List + } + return nil +} + +type WorkspaceDependencies struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Dependencies map[string]*LockfilePackageList `protobuf:"bytes,1,rep,name=dependencies,proto3" json:"dependencies,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *WorkspaceDependencies) Reset() { + *x = WorkspaceDependencies{} + if protoimpl.UnsafeEnabled { + mi := &file_turborepo_ffi_messages_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *WorkspaceDependencies) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*WorkspaceDependencies) ProtoMessage() {} + +func (x *WorkspaceDependencies) ProtoReflect() protoreflect.Message { + mi := &file_turborepo_ffi_messages_proto_msgTypes[11] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use WorkspaceDependencies.ProtoReflect.Descriptor instead. +func (*WorkspaceDependencies) Descriptor() ([]byte, []int) { + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{11} +} + +func (x *WorkspaceDependencies) GetDependencies() map[string]*LockfilePackageList { + if x != nil { + return x.Dependencies + } + return nil +} + type TransitiveDepsRequest struct { state protoimpl.MessageState sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Contents []byte `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` - WorkspaceDir string `protobuf:"bytes,2,opt,name=workspace_dir,json=workspaceDir,proto3" json:"workspace_dir,omitempty"` - UnresolvedDeps map[string]string `protobuf:"bytes,3,rep,name=unresolved_deps,json=unresolvedDeps,proto3" json:"unresolved_deps,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` + Contents []byte `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` + PackageManager PackageManager `protobuf:"varint,2,opt,name=package_manager,json=packageManager,proto3,enum=PackageManager" json:"package_manager,omitempty"` + Workspaces map[string]*PackageDependencyList `protobuf:"bytes,3,rep,name=workspaces,proto3" json:"workspaces,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` } func (x *TransitiveDepsRequest) Reset() { *x = TransitiveDepsRequest{} if protoimpl.UnsafeEnabled { - mi := &file_turborepo_ffi_messages_proto_msgTypes[9] + mi := &file_turborepo_ffi_messages_proto_msgTypes[12] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -632,7 +824,7 @@ func (x *TransitiveDepsRequest) String() string { func (*TransitiveDepsRequest) ProtoMessage() {} func (x *TransitiveDepsRequest) ProtoReflect() protoreflect.Message { - mi := &file_turborepo_ffi_messages_proto_msgTypes[9] + mi := &file_turborepo_ffi_messages_proto_msgTypes[12] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -645,7 +837,7 @@ func (x *TransitiveDepsRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use TransitiveDepsRequest.ProtoReflect.Descriptor instead. func (*TransitiveDepsRequest) Descriptor() ([]byte, []int) { - return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{9} + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{12} } func (x *TransitiveDepsRequest) GetContents() []byte { @@ -655,16 +847,16 @@ func (x *TransitiveDepsRequest) GetContents() []byte { return nil } -func (x *TransitiveDepsRequest) GetWorkspaceDir() string { +func (x *TransitiveDepsRequest) GetPackageManager() PackageManager { if x != nil { - return x.WorkspaceDir + return x.PackageManager } - return "" + return PackageManager_NPM } -func (x *TransitiveDepsRequest) GetUnresolvedDeps() map[string]string { +func (x *TransitiveDepsRequest) GetWorkspaces() map[string]*PackageDependencyList { if x != nil { - return x.UnresolvedDeps + return x.Workspaces } return nil } @@ -675,7 +867,7 @@ type TransitiveDepsResponse struct { unknownFields protoimpl.UnknownFields // Types that are assignable to Response: - // *TransitiveDepsResponse_Packages + // *TransitiveDepsResponse_Dependencies // *TransitiveDepsResponse_Error Response isTransitiveDepsResponse_Response `protobuf_oneof:"response"` } @@ -683,7 +875,7 @@ type TransitiveDepsResponse struct { func (x *TransitiveDepsResponse) Reset() { *x = TransitiveDepsResponse{} if protoimpl.UnsafeEnabled { - mi := &file_turborepo_ffi_messages_proto_msgTypes[10] + mi := &file_turborepo_ffi_messages_proto_msgTypes[13] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -696,7 +888,7 @@ func (x *TransitiveDepsResponse) String() string { func (*TransitiveDepsResponse) ProtoMessage() {} func (x *TransitiveDepsResponse) ProtoReflect() protoreflect.Message { - mi := &file_turborepo_ffi_messages_proto_msgTypes[10] + mi := &file_turborepo_ffi_messages_proto_msgTypes[13] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -709,7 +901,7 @@ func (x *TransitiveDepsResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use TransitiveDepsResponse.ProtoReflect.Descriptor instead. func (*TransitiveDepsResponse) Descriptor() ([]byte, []int) { - return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{10} + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{13} } func (m *TransitiveDepsResponse) GetResponse() isTransitiveDepsResponse_Response { @@ -719,9 +911,9 @@ func (m *TransitiveDepsResponse) GetResponse() isTransitiveDepsResponse_Response return nil } -func (x *TransitiveDepsResponse) GetPackages() *LockfilePackageList { - if x, ok := x.GetResponse().(*TransitiveDepsResponse_Packages); ok { - return x.Packages +func (x *TransitiveDepsResponse) GetDependencies() *WorkspaceDependencies { + if x, ok := x.GetResponse().(*TransitiveDepsResponse_Dependencies); ok { + return x.Dependencies } return nil } @@ -737,15 +929,15 @@ type isTransitiveDepsResponse_Response interface { isTransitiveDepsResponse_Response() } -type TransitiveDepsResponse_Packages struct { - Packages *LockfilePackageList `protobuf:"bytes,1,opt,name=packages,proto3,oneof"` +type TransitiveDepsResponse_Dependencies struct { + Dependencies *WorkspaceDependencies `protobuf:"bytes,1,opt,name=dependencies,proto3,oneof"` } type TransitiveDepsResponse_Error struct { Error string `protobuf:"bytes,2,opt,name=error,proto3,oneof"` } -func (*TransitiveDepsResponse_Packages) isTransitiveDepsResponse_Response() {} +func (*TransitiveDepsResponse_Dependencies) isTransitiveDepsResponse_Response() {} func (*TransitiveDepsResponse_Error) isTransitiveDepsResponse_Response() {} @@ -762,7 +954,7 @@ type LockfilePackage struct { func (x *LockfilePackage) Reset() { *x = LockfilePackage{} if protoimpl.UnsafeEnabled { - mi := &file_turborepo_ffi_messages_proto_msgTypes[11] + mi := &file_turborepo_ffi_messages_proto_msgTypes[14] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -775,7 +967,7 @@ func (x *LockfilePackage) String() string { func (*LockfilePackage) ProtoMessage() {} func (x *LockfilePackage) ProtoReflect() protoreflect.Message { - mi := &file_turborepo_ffi_messages_proto_msgTypes[11] + mi := &file_turborepo_ffi_messages_proto_msgTypes[14] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -788,7 +980,7 @@ func (x *LockfilePackage) ProtoReflect() protoreflect.Message { // Deprecated: Use LockfilePackage.ProtoReflect.Descriptor instead. func (*LockfilePackage) Descriptor() ([]byte, []int) { - return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{11} + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{14} } func (x *LockfilePackage) GetKey() string { @@ -823,7 +1015,7 @@ type LockfilePackageList struct { func (x *LockfilePackageList) Reset() { *x = LockfilePackageList{} if protoimpl.UnsafeEnabled { - mi := &file_turborepo_ffi_messages_proto_msgTypes[12] + mi := &file_turborepo_ffi_messages_proto_msgTypes[15] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -836,7 +1028,7 @@ func (x *LockfilePackageList) String() string { func (*LockfilePackageList) ProtoMessage() {} func (x *LockfilePackageList) ProtoReflect() protoreflect.Message { - mi := &file_turborepo_ffi_messages_proto_msgTypes[12] + mi := &file_turborepo_ffi_messages_proto_msgTypes[15] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -849,7 +1041,7 @@ func (x *LockfilePackageList) ProtoReflect() protoreflect.Message { // Deprecated: Use LockfilePackageList.ProtoReflect.Descriptor instead. func (*LockfilePackageList) Descriptor() ([]byte, []int) { - return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{12} + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{15} } func (x *LockfilePackageList) GetList() []*LockfilePackage { @@ -864,15 +1056,16 @@ type SubgraphRequest struct { sizeCache protoimpl.SizeCache unknownFields protoimpl.UnknownFields - Contents []byte `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` - Workspaces []string `protobuf:"bytes,2,rep,name=workspaces,proto3" json:"workspaces,omitempty"` - Packages []string `protobuf:"bytes,3,rep,name=packages,proto3" json:"packages,omitempty"` + Contents []byte `protobuf:"bytes,1,opt,name=contents,proto3" json:"contents,omitempty"` + PackageManager string `protobuf:"bytes,2,opt,name=package_manager,json=packageManager,proto3" json:"package_manager,omitempty"` + Workspaces []string `protobuf:"bytes,3,rep,name=workspaces,proto3" json:"workspaces,omitempty"` + Packages []string `protobuf:"bytes,4,rep,name=packages,proto3" json:"packages,omitempty"` } func (x *SubgraphRequest) Reset() { *x = SubgraphRequest{} if protoimpl.UnsafeEnabled { - mi := &file_turborepo_ffi_messages_proto_msgTypes[13] + mi := &file_turborepo_ffi_messages_proto_msgTypes[16] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -885,7 +1078,7 @@ func (x *SubgraphRequest) String() string { func (*SubgraphRequest) ProtoMessage() {} func (x *SubgraphRequest) ProtoReflect() protoreflect.Message { - mi := &file_turborepo_ffi_messages_proto_msgTypes[13] + mi := &file_turborepo_ffi_messages_proto_msgTypes[16] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -898,7 +1091,7 @@ func (x *SubgraphRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use SubgraphRequest.ProtoReflect.Descriptor instead. func (*SubgraphRequest) Descriptor() ([]byte, []int) { - return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{13} + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{16} } func (x *SubgraphRequest) GetContents() []byte { @@ -908,6 +1101,13 @@ func (x *SubgraphRequest) GetContents() []byte { return nil } +func (x *SubgraphRequest) GetPackageManager() string { + if x != nil { + return x.PackageManager + } + return "" +} + func (x *SubgraphRequest) GetWorkspaces() []string { if x != nil { return x.Workspaces @@ -936,7 +1136,7 @@ type SubgraphResponse struct { func (x *SubgraphResponse) Reset() { *x = SubgraphResponse{} if protoimpl.UnsafeEnabled { - mi := &file_turborepo_ffi_messages_proto_msgTypes[14] + mi := &file_turborepo_ffi_messages_proto_msgTypes[17] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -949,7 +1149,7 @@ func (x *SubgraphResponse) String() string { func (*SubgraphResponse) ProtoMessage() {} func (x *SubgraphResponse) ProtoReflect() protoreflect.Message { - mi := &file_turborepo_ffi_messages_proto_msgTypes[14] + mi := &file_turborepo_ffi_messages_proto_msgTypes[17] if protoimpl.UnsafeEnabled && x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -962,7 +1162,7 @@ func (x *SubgraphResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use SubgraphResponse.ProtoReflect.Descriptor instead. func (*SubgraphResponse) Descriptor() ([]byte, []int) { - return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{14} + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{17} } func (m *SubgraphResponse) GetResponse() isSubgraphResponse_Response { @@ -1058,51 +1258,78 @@ var file_turborepo_ffi_messages_proto_rawDesc = []byte{ 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x07, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, - 0x6e, 0x73, 0x65, 0x22, 0xf0, 0x01, 0x0a, 0x15, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, - 0x76, 0x65, 0x44, 0x65, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, - 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x23, 0x0a, 0x0d, 0x77, 0x6f, 0x72, - 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x5f, 0x64, 0x69, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, - 0x52, 0x0c, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x44, 0x69, 0x72, 0x12, 0x53, - 0x0a, 0x0f, 0x75, 0x6e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x5f, 0x64, 0x65, 0x70, - 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x2a, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, - 0x74, 0x69, 0x76, 0x65, 0x44, 0x65, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, - 0x55, 0x6e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x44, 0x65, 0x70, 0x73, 0x45, 0x6e, - 0x74, 0x72, 0x79, 0x52, 0x0e, 0x75, 0x6e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, 0x64, 0x44, - 0x65, 0x70, 0x73, 0x1a, 0x41, 0x0a, 0x13, 0x55, 0x6e, 0x72, 0x65, 0x73, 0x6f, 0x6c, 0x76, 0x65, - 0x64, 0x44, 0x65, 0x70, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, - 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, 0x0a, 0x05, - 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x76, 0x61, 0x6c, - 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x70, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, - 0x74, 0x69, 0x76, 0x65, 0x44, 0x65, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, - 0x12, 0x32, 0x0a, 0x08, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, - 0x28, 0x0b, 0x32, 0x14, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x48, 0x00, 0x52, 0x08, 0x70, 0x61, 0x63, 0x6b, - 0x61, 0x67, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, - 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x0a, 0x0a, 0x08, - 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x53, 0x0a, 0x0f, 0x4c, 0x6f, 0x63, 0x6b, - 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x12, 0x10, 0x0a, 0x03, 0x6b, - 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x18, 0x0a, - 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x07, - 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, - 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x22, 0x3b, 0x0a, - 0x13, 0x4c, 0x6f, 0x63, 0x6b, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, - 0x4c, 0x69, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, - 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, 0x69, 0x0a, 0x0f, 0x53, 0x75, - 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, - 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, - 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x1e, 0x0a, 0x0a, 0x77, 0x6f, 0x72, - 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x02, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x77, - 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x63, - 0x6b, 0x61, 0x67, 0x65, 0x73, 0x22, 0x54, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, - 0x68, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x08, 0x63, 0x6f, 0x6e, - 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, 0x63, - 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, - 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, - 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x42, 0x0b, 0x5a, 0x09, 0x66, - 0x66, 0x69, 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6e, 0x73, 0x65, 0x22, 0x3d, 0x0a, 0x11, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x44, 0x65, + 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x14, 0x0a, 0x05, + 0x72, 0x61, 0x6e, 0x67, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x05, 0x72, 0x61, 0x6e, + 0x67, 0x65, 0x22, 0x3f, 0x0a, 0x15, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x44, 0x65, 0x70, + 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x26, 0x0a, 0x04, 0x6c, + 0x69, 0x73, 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x12, 0x2e, 0x50, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x79, 0x52, 0x04, 0x6c, + 0x69, 0x73, 0x74, 0x22, 0xbc, 0x01, 0x0a, 0x15, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, + 0x65, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x4c, 0x0a, + 0x0c, 0x64, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, + 0x03, 0x28, 0x0b, 0x32, 0x28, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x44, + 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x2e, 0x44, 0x65, 0x70, 0x65, + 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0c, 0x64, + 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x1a, 0x55, 0x0a, 0x11, 0x44, + 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x2a, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x14, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, + 0x38, 0x01, 0x22, 0x8c, 0x02, 0x0a, 0x15, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x76, + 0x65, 0x44, 0x65, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, + 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x38, 0x0a, 0x0f, 0x70, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, + 0x0e, 0x32, 0x0f, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x52, 0x0e, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x12, 0x46, 0x0a, 0x0a, 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, + 0x18, 0x03, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x26, 0x2e, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, + 0x69, 0x76, 0x65, 0x44, 0x65, 0x70, 0x73, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x2e, 0x57, + 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x0a, + 0x77, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x1a, 0x55, 0x0a, 0x0f, 0x57, 0x6f, + 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, 0x0a, + 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, + 0x2c, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x16, + 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x44, 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, + 0x63, 0x79, 0x4c, 0x69, 0x73, 0x74, 0x52, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, + 0x01, 0x22, 0x7a, 0x0a, 0x16, 0x54, 0x72, 0x61, 0x6e, 0x73, 0x69, 0x74, 0x69, 0x76, 0x65, 0x44, + 0x65, 0x70, 0x73, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x3c, 0x0a, 0x0c, 0x64, + 0x65, 0x70, 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, + 0x0b, 0x32, 0x16, 0x2e, 0x57, 0x6f, 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x44, 0x65, 0x70, + 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x48, 0x00, 0x52, 0x0c, 0x64, 0x65, 0x70, + 0x65, 0x6e, 0x64, 0x65, 0x6e, 0x63, 0x69, 0x65, 0x73, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, + 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, + 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x22, 0x53, 0x0a, + 0x0f, 0x4c, 0x6f, 0x63, 0x6b, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, + 0x12, 0x10, 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, + 0x65, 0x79, 0x12, 0x18, 0x0a, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x07, 0x76, 0x65, 0x72, 0x73, 0x69, 0x6f, 0x6e, 0x12, 0x14, 0x0a, 0x05, + 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x05, 0x66, 0x6f, 0x75, + 0x6e, 0x64, 0x22, 0x3b, 0x0a, 0x13, 0x4c, 0x6f, 0x63, 0x6b, 0x66, 0x69, 0x6c, 0x65, 0x50, 0x61, + 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4c, 0x69, 0x73, 0x74, 0x12, 0x24, 0x0a, 0x04, 0x6c, 0x69, 0x73, + 0x74, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x10, 0x2e, 0x4c, 0x6f, 0x63, 0x6b, 0x66, 0x69, + 0x6c, 0x65, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x04, 0x6c, 0x69, 0x73, 0x74, 0x22, + 0x92, 0x01, 0x0a, 0x0f, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, 0x52, 0x65, 0x71, 0x75, + 0x65, 0x73, 0x74, 0x12, 0x1a, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x18, + 0x01, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x08, 0x63, 0x6f, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, + 0x27, 0x0a, 0x0f, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x6d, 0x61, 0x6e, 0x61, 0x67, + 0x65, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, 0x0e, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, + 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x1e, 0x0a, 0x0a, 0x77, 0x6f, 0x72, 0x6b, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x18, 0x03, 0x20, 0x03, 0x28, 0x09, 0x52, 0x0a, 0x77, 0x6f, + 0x72, 0x6b, 0x73, 0x70, 0x61, 0x63, 0x65, 0x73, 0x12, 0x1a, 0x0a, 0x08, 0x70, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x73, 0x18, 0x04, 0x20, 0x03, 0x28, 0x09, 0x52, 0x08, 0x70, 0x61, 0x63, 0x6b, + 0x61, 0x67, 0x65, 0x73, 0x22, 0x54, 0x0a, 0x10, 0x53, 0x75, 0x62, 0x67, 0x72, 0x61, 0x70, 0x68, + 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x1c, 0x0a, 0x08, 0x63, 0x6f, 0x6e, 0x74, + 0x65, 0x6e, 0x74, 0x73, 0x18, 0x01, 0x20, 0x01, 0x28, 0x0c, 0x48, 0x00, 0x52, 0x08, 0x63, 0x6f, + 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x73, 0x12, 0x16, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, + 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x0a, + 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x2a, 0x19, 0x0a, 0x0e, 0x50, 0x61, + 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, + 0x4e, 0x50, 0x4d, 0x10, 0x00, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, 0x69, 0x2f, 0x70, 0x72, 0x6f, + 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1117,36 +1344,47 @@ func file_turborepo_ffi_messages_proto_rawDescGZIP() []byte { return file_turborepo_ffi_messages_proto_rawDescData } -var file_turborepo_ffi_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 16) +var file_turborepo_ffi_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_turborepo_ffi_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 20) var file_turborepo_ffi_messages_proto_goTypes = []interface{}{ - (*TurboDataDirResp)(nil), // 0: TurboDataDirResp - (*GlobReq)(nil), // 1: GlobReq - (*GlobResp)(nil), // 2: GlobResp - (*GlobRespList)(nil), // 3: GlobRespList - (*ChangedFilesReq)(nil), // 4: ChangedFilesReq - (*ChangedFilesResp)(nil), // 5: ChangedFilesResp - (*ChangedFilesList)(nil), // 6: ChangedFilesList - (*PreviousContentReq)(nil), // 7: PreviousContentReq - (*PreviousContentResp)(nil), // 8: PreviousContentResp - (*TransitiveDepsRequest)(nil), // 9: TransitiveDepsRequest - (*TransitiveDepsResponse)(nil), // 10: TransitiveDepsResponse - (*LockfilePackage)(nil), // 11: LockfilePackage - (*LockfilePackageList)(nil), // 12: LockfilePackageList - (*SubgraphRequest)(nil), // 13: SubgraphRequest - (*SubgraphResponse)(nil), // 14: SubgraphResponse - nil, // 15: TransitiveDepsRequest.UnresolvedDepsEntry + (PackageManager)(0), // 0: PackageManager + (*TurboDataDirResp)(nil), // 1: TurboDataDirResp + (*GlobReq)(nil), // 2: GlobReq + (*GlobResp)(nil), // 3: GlobResp + (*GlobRespList)(nil), // 4: GlobRespList + (*ChangedFilesReq)(nil), // 5: ChangedFilesReq + (*ChangedFilesResp)(nil), // 6: ChangedFilesResp + (*ChangedFilesList)(nil), // 7: ChangedFilesList + (*PreviousContentReq)(nil), // 8: PreviousContentReq + (*PreviousContentResp)(nil), // 9: PreviousContentResp + (*PackageDependency)(nil), // 10: PackageDependency + (*PackageDependencyList)(nil), // 11: PackageDependencyList + (*WorkspaceDependencies)(nil), // 12: WorkspaceDependencies + (*TransitiveDepsRequest)(nil), // 13: TransitiveDepsRequest + (*TransitiveDepsResponse)(nil), // 14: TransitiveDepsResponse + (*LockfilePackage)(nil), // 15: LockfilePackage + (*LockfilePackageList)(nil), // 16: LockfilePackageList + (*SubgraphRequest)(nil), // 17: SubgraphRequest + (*SubgraphResponse)(nil), // 18: SubgraphResponse + nil, // 19: WorkspaceDependencies.DependenciesEntry + nil, // 20: TransitiveDepsRequest.WorkspacesEntry } var file_turborepo_ffi_messages_proto_depIdxs = []int32{ - 3, // 0: GlobResp.files:type_name -> GlobRespList - 6, // 1: ChangedFilesResp.files:type_name -> ChangedFilesList - 15, // 2: TransitiveDepsRequest.unresolved_deps:type_name -> TransitiveDepsRequest.UnresolvedDepsEntry - 12, // 3: TransitiveDepsResponse.packages:type_name -> LockfilePackageList - 11, // 4: LockfilePackageList.list:type_name -> LockfilePackage - 5, // [5:5] is the sub-list for method output_type - 5, // [5:5] is the sub-list for method input_type - 5, // [5:5] is the sub-list for extension type_name - 5, // [5:5] is the sub-list for extension extendee - 0, // [0:5] is the sub-list for field type_name + 4, // 0: GlobResp.files:type_name -> GlobRespList + 7, // 1: ChangedFilesResp.files:type_name -> ChangedFilesList + 10, // 2: PackageDependencyList.list:type_name -> PackageDependency + 19, // 3: WorkspaceDependencies.dependencies:type_name -> WorkspaceDependencies.DependenciesEntry + 0, // 4: TransitiveDepsRequest.package_manager:type_name -> PackageManager + 20, // 5: TransitiveDepsRequest.workspaces:type_name -> TransitiveDepsRequest.WorkspacesEntry + 12, // 6: TransitiveDepsResponse.dependencies:type_name -> WorkspaceDependencies + 15, // 7: LockfilePackageList.list:type_name -> LockfilePackage + 16, // 8: WorkspaceDependencies.DependenciesEntry.value:type_name -> LockfilePackageList + 11, // 9: TransitiveDepsRequest.WorkspacesEntry.value:type_name -> PackageDependencyList + 10, // [10:10] is the sub-list for method output_type + 10, // [10:10] is the sub-list for method input_type + 10, // [10:10] is the sub-list for extension type_name + 10, // [10:10] is the sub-list for extension extendee + 0, // [0:10] is the sub-list for field type_name } func init() { file_turborepo_ffi_messages_proto_init() } @@ -1264,7 +1502,7 @@ func file_turborepo_ffi_messages_proto_init() { } } file_turborepo_ffi_messages_proto_msgTypes[9].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TransitiveDepsRequest); i { + switch v := v.(*PackageDependency); i { case 0: return &v.state case 1: @@ -1276,7 +1514,7 @@ func file_turborepo_ffi_messages_proto_init() { } } file_turborepo_ffi_messages_proto_msgTypes[10].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*TransitiveDepsResponse); i { + switch v := v.(*PackageDependencyList); i { case 0: return &v.state case 1: @@ -1288,7 +1526,7 @@ func file_turborepo_ffi_messages_proto_init() { } } file_turborepo_ffi_messages_proto_msgTypes[11].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LockfilePackage); i { + switch v := v.(*WorkspaceDependencies); i { case 0: return &v.state case 1: @@ -1300,7 +1538,7 @@ func file_turborepo_ffi_messages_proto_init() { } } file_turborepo_ffi_messages_proto_msgTypes[12].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*LockfilePackageList); i { + switch v := v.(*TransitiveDepsRequest); i { case 0: return &v.state case 1: @@ -1312,7 +1550,7 @@ func file_turborepo_ffi_messages_proto_init() { } } file_turborepo_ffi_messages_proto_msgTypes[13].Exporter = func(v interface{}, i int) interface{} { - switch v := v.(*SubgraphRequest); i { + switch v := v.(*TransitiveDepsResponse); i { case 0: return &v.state case 1: @@ -1324,6 +1562,42 @@ func file_turborepo_ffi_messages_proto_init() { } } file_turborepo_ffi_messages_proto_msgTypes[14].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LockfilePackage); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_turborepo_ffi_messages_proto_msgTypes[15].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*LockfilePackageList); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_turborepo_ffi_messages_proto_msgTypes[16].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*SubgraphRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_turborepo_ffi_messages_proto_msgTypes[17].Exporter = func(v interface{}, i int) interface{} { switch v := v.(*SubgraphResponse); i { case 0: return &v.state @@ -1349,11 +1623,11 @@ func file_turborepo_ffi_messages_proto_init() { (*PreviousContentResp_Content)(nil), (*PreviousContentResp_Error)(nil), } - file_turborepo_ffi_messages_proto_msgTypes[10].OneofWrappers = []interface{}{ - (*TransitiveDepsResponse_Packages)(nil), + file_turborepo_ffi_messages_proto_msgTypes[13].OneofWrappers = []interface{}{ + (*TransitiveDepsResponse_Dependencies)(nil), (*TransitiveDepsResponse_Error)(nil), } - file_turborepo_ffi_messages_proto_msgTypes[14].OneofWrappers = []interface{}{ + file_turborepo_ffi_messages_proto_msgTypes[17].OneofWrappers = []interface{}{ (*SubgraphResponse_Contents)(nil), (*SubgraphResponse_Error)(nil), } @@ -1362,13 +1636,14 @@ func file_turborepo_ffi_messages_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_turborepo_ffi_messages_proto_rawDesc, - NumEnums: 0, - NumMessages: 16, + NumEnums: 1, + NumMessages: 20, NumExtensions: 0, NumServices: 0, }, GoTypes: file_turborepo_ffi_messages_proto_goTypes, DependencyIndexes: file_turborepo_ffi_messages_proto_depIdxs, + EnumInfos: file_turborepo_ffi_messages_proto_enumTypes, MessageInfos: file_turborepo_ffi_messages_proto_msgTypes, }.Build() File_turborepo_ffi_messages_proto = out.File diff --git a/cli/internal/fs/package_json.go b/cli/internal/fs/package_json.go index 883f7a4ac7df0..a8efe3ed5220a 100644 --- a/cli/internal/fs/package_json.go +++ b/cli/internal/fs/package_json.go @@ -3,8 +3,10 @@ package fs import ( "bytes" "encoding/json" + "sort" "sync" + mapset "github.com/deckarep/golang-set" "github.com/vercel/turbo/cli/internal/lockfile" "github.com/vercel/turbo/cli/internal/turbopath" ) @@ -120,6 +122,24 @@ func MarshalPackageJSON(pkgJSON *PackageJSON) ([]byte, error) { return b.Bytes(), nil } +// SetExternalDeps sets TransitiveDeps and populates ExternalDepsHash +func (p *PackageJSON) SetExternalDeps(externalDeps mapset.Set) error { + p.Mu.Lock() + defer p.Mu.Unlock() + p.TransitiveDeps = make([]lockfile.Package, 0, externalDeps.Cardinality()) + for _, dependency := range externalDeps.ToSlice() { + dependency := dependency.(lockfile.Package) + p.TransitiveDeps = append(p.TransitiveDeps, dependency) + } + sort.Sort(lockfile.ByKey(p.TransitiveDeps)) + hashOfExternalDeps, err := HashObject(p.TransitiveDeps) + if err != nil { + return err + } + p.ExternalDepsHash = hashOfExternalDeps + return nil +} + func isEmpty(value interface{}) bool { if value == nil { return true diff --git a/cli/internal/fs/package_json_test.go b/cli/internal/fs/package_json_test.go index 3c1662008aa6f..246e3cbfd7be2 100644 --- a/cli/internal/fs/package_json_test.go +++ b/cli/internal/fs/package_json_test.go @@ -3,6 +3,7 @@ package fs import ( "testing" + mapset "github.com/deckarep/golang-set" "gotest.tools/v3/assert" ) @@ -157,6 +158,15 @@ func Test_MarshalPackageJSON(t *testing.T) { } } +func Test_SetExternalDepsWithEmptySet(t *testing.T) { + pkg := &PackageJSON{} + err := pkg.SetExternalDeps(mapset.NewSet()) + assert.NilError(t, err) + assert.Assert(t, pkg.TransitiveDeps != nil) + assert.Equal(t, len(pkg.TransitiveDeps), 0) + assert.DeepEqual(t, pkg.ExternalDepsHash, "ccab0b28617f1f56") +} + // Asserts that the data section of two PackageJSON structs are equal func assertPackageJSONEqual(t *testing.T, x *PackageJSON, y *PackageJSON) { t.Helper() diff --git a/cli/internal/lockfile/lockfile.go b/cli/internal/lockfile/lockfile.go index bb36eda93f419..b24deee88f818 100644 --- a/cli/internal/lockfile/lockfile.go +++ b/cli/internal/lockfile/lockfile.go @@ -8,6 +8,7 @@ import ( "sort" mapset "github.com/deckarep/golang-set" + "github.com/vercel/turbo/cli/internal/ffi" "github.com/vercel/turbo/cli/internal/turbopath" "golang.org/x/sync/errgroup" ) @@ -61,17 +62,45 @@ func (p ByKey) Less(i, j int) bool { var _ (sort.Interface) = (*ByKey)(nil) -// TransitiveClosure the set of all lockfile keys that pkg depends on -func TransitiveClosure( - workspaceDir turbopath.AnchoredUnixPath, - unresolvedDeps map[string]string, +type closureMsg struct { + workspace turbopath.AnchoredUnixPath + closure mapset.Set +} + +// AllTransitiveClosures computes closures for all workspaces +func AllTransitiveClosures( + workspaces map[turbopath.AnchoredUnixPath]map[string]string, lockFile Lockfile, -) (mapset.Set, error) { +) (map[turbopath.AnchoredUnixPath]mapset.Set, error) { if lf, ok := lockFile.(*NpmLockfile); ok { // We special case as Rust implementations have their own dep crawl - return npmTransitiveDeps(lf, workspaceDir, unresolvedDeps) + return rustTransitiveDeps(lf.contents, "npm", workspaces) } - return transitiveClosure(workspaceDir, unresolvedDeps, lockFile) + + g := new(errgroup.Group) + c := make(chan closureMsg, len(workspaces)) + closures := make(map[turbopath.AnchoredUnixPath]mapset.Set, len(workspaces)) + for workspace, deps := range workspaces { + workspace := workspace + deps := deps + g.Go(func() error { + closure, err := transitiveClosure(workspace, deps, lockFile) + if err != nil { + return err + } + c <- closureMsg{workspace: workspace, closure: closure} + return nil + }) + } + err := g.Wait() + close(c) + if err != nil { + return nil, err + } + for msg := range c { + closures[msg.workspace] = msg.closure + } + return closures, nil } func transitiveClosure( @@ -133,3 +162,28 @@ func transitiveClosureHelper( }) } } + +func rustTransitiveDeps(content []byte, packageManager string, workspaces map[turbopath.AnchoredUnixPath]map[string]string) (map[turbopath.AnchoredUnixPath]mapset.Set, error) { + processedWorkspaces := make(map[string]map[string]string, len(workspaces)) + for workspacePath, workspace := range workspaces { + processedWorkspaces[workspacePath.ToString()] = workspace + } + workspaceDeps, err := ffi.TransitiveDeps(content, packageManager, processedWorkspaces) + if err != nil { + return nil, err + } + resolvedWorkspaces := make(map[turbopath.AnchoredUnixPath]mapset.Set, len(workspaceDeps)) + for workspace, dependencies := range workspaceDeps { + depsSet := mapset.NewSet() + for _, pkg := range dependencies.GetList() { + depsSet.Add(Package{ + Found: pkg.Found, + Key: pkg.Key, + Version: pkg.Version, + }) + } + workspacePath := turbopath.AnchoredUnixPath(workspace) + resolvedWorkspaces[workspacePath] = depsSet + } + return resolvedWorkspaces, nil +} diff --git a/cli/internal/lockfile/lockfile_test.go b/cli/internal/lockfile/lockfile_test.go index 7c666cc4f3000..1ffedc58b1a2d 100644 --- a/cli/internal/lockfile/lockfile_test.go +++ b/cli/internal/lockfile/lockfile_test.go @@ -1,9 +1,12 @@ package lockfile import ( + "io" + "reflect" "sort" "testing" + "github.com/vercel/turbo/cli/internal/turbopath" "gotest.tools/v3/assert" ) @@ -23,3 +26,45 @@ func Test_ByKeySortIsStable(t *testing.T) { assert.DeepEqual(t, packagesA, packagesB) } + +type mockLockfile struct{} + +func (m *mockLockfile) ResolvePackage(_ turbopath.AnchoredUnixPath, _ string, _ string) (Package, error) { + panic("unimplemented") +} + +func (m *mockLockfile) AllDependencies(_ string) (map[string]string, bool) { + panic("unimplemented") +} + +func (m *mockLockfile) Subgraph(_ []turbopath.AnchoredSystemPath, _ []string) (Lockfile, error) { + panic("unimplemented") +} + +func (m *mockLockfile) Encode(_ io.Writer) error { + panic("unimplemented") +} + +func (m *mockLockfile) Patches() []turbopath.AnchoredUnixPath { + panic("unimplemented") +} + +func (m *mockLockfile) GlobalChange(_ Lockfile) bool { + panic("unimplemented") +} + +var _ (Lockfile) = (*mockLockfile)(nil) + +func Test_AllTransitiveClosureReturnsEmptySets(t *testing.T) { + closures, err := AllTransitiveClosures(map[turbopath.AnchoredUnixPath]map[string]string{ + turbopath.AnchoredUnixPath("."): {}, + turbopath.AnchoredUnixPath("a"): {}, + turbopath.AnchoredUnixPath("b"): {}, + }, &mockLockfile{}) + assert.NilError(t, err) + assert.Assert(t, len(closures) == 3) + for _, closure := range closures { + assert.Assert(t, closure != nil && !reflect.ValueOf(closure).IsNil()) + assert.Equal(t, closure.Cardinality(), 0) + } +} diff --git a/cli/internal/lockfile/npm_lockfile.go b/cli/internal/lockfile/npm_lockfile.go index 67cd32af63897..dceb560f4c782 100644 --- a/cli/internal/lockfile/npm_lockfile.go +++ b/cli/internal/lockfile/npm_lockfile.go @@ -4,7 +4,6 @@ import ( "encoding/json" "io" - mapset "github.com/deckarep/golang-set" "github.com/vercel/turbo/cli/internal/ffi" "github.com/vercel/turbo/cli/internal/turbopath" ) @@ -85,23 +84,3 @@ var _ (Lockfile) = (*NpmLockfile)(nil) func DecodeNpmLockfile(contents []byte) (Lockfile, error) { return &NpmLockfile{contents: contents}, nil } - -func npmTransitiveDeps(lockfile *NpmLockfile, workspacePath turbopath.AnchoredUnixPath, unresolvedDeps map[string]string) (mapset.Set, error) { - pkgDir := workspacePath.ToString() - - packages, err := ffi.NpmTransitiveDeps(lockfile.contents, pkgDir, unresolvedDeps) - if err != nil { - return nil, err - } - - deps := make([]interface{}, len(packages)) - for i, pkg := range packages { - deps[i] = Package{ - Found: pkg.Found, - Key: pkg.Key, - Version: pkg.Version, - } - } - - return mapset.NewSetFromSlice(deps), nil -} diff --git a/cli/internal/lockfile/npm_lockfile_test.go b/cli/internal/lockfile/npm_lockfile_test.go new file mode 100644 index 0000000000000..5dfd327b08f7b --- /dev/null +++ b/cli/internal/lockfile/npm_lockfile_test.go @@ -0,0 +1,74 @@ +package lockfile + +import ( + "os" + "testing" + + "github.com/vercel/turbo/cli/internal/turbopath" + "gotest.tools/v3/assert" +) + +func getRustFixture(t *testing.T, fixture string) []byte { + defaultCwd, err := os.Getwd() + assert.NilError(t, err, "getRustFixture") + cwd := turbopath.AbsoluteSystemPath(defaultCwd) + lockfilePath := cwd.UntypedJoin("../../../crates/turborepo-lockfiles/fixtures", fixture) + if !lockfilePath.FileExists() { + t.Errorf("unable to find 'turborepo-lockfiles/fixtures/%s'", fixture) + } + bytes, err := os.ReadFile(lockfilePath.ToString()) + assert.NilError(t, err, "unable to read fixture") + return bytes +} + +func getNpmFixture(t *testing.T, fixture string) Lockfile { + bytes := getRustFixture(t, fixture) + lf, err := DecodeNpmLockfile(bytes) + assert.NilError(t, err) + return lf +} + +func TestAllDependenciesNpm(t *testing.T) { + lf := getNpmFixture(t, "npm-lock.json") + closures, err := AllTransitiveClosures(map[turbopath.AnchoredUnixPath]map[string]string{ + turbopath.AnchoredUnixPath(""): { + "turbo": "latest", + "prettier": "latest", + }, + turbopath.AnchoredUnixPath("apps/web"): { + "lodash": "^4.17.21", + "next": "12.3.0", + }, + }, lf) + assert.NilError(t, err) + assert.Equal(t, len(closures), 2) + rootClosure := closures[turbopath.AnchoredUnixPath("")] + webClosure := closures[turbopath.AnchoredUnixPath("apps/web")] + + assert.Assert(t, rootClosure.Contains(Package{ + Key: "node_modules/turbo", + Version: "1.5.5", + Found: true, + })) + assert.Assert(t, rootClosure.Contains(Package{ + Key: "node_modules/turbo-darwin-64", + Version: "1.5.5", + Found: true, + })) + + assert.Assert(t, webClosure.Contains(Package{ + Key: "apps/web/node_modules/lodash", + Version: "4.17.21", + Found: true, + })) + assert.Assert(t, webClosure.Contains(Package{ + Key: "node_modules/next", + Version: "12.3.0", + Found: true, + })) + assert.Assert(t, webClosure.Contains(Package{ + Key: "node_modules/postcss", + Version: "8.4.14", + Found: true, + })) +} diff --git a/crates/turborepo-ffi/messages.proto b/crates/turborepo-ffi/messages.proto index 8749d3b9b527d..31bd10d514d1b 100644 --- a/crates/turborepo-ffi/messages.proto +++ b/crates/turborepo-ffi/messages.proto @@ -54,15 +54,32 @@ message PreviousContentResp { } } +enum PackageManager { + NPM = 0; +} + +message PackageDependency { + string name = 1; + string range = 2; +} + +message PackageDependencyList { + repeated PackageDependency list = 1; +} + +message WorkspaceDependencies { + map dependencies = 1; +} + message TransitiveDepsRequest { bytes contents = 1; - string workspace_dir = 2; - map unresolved_deps = 3; + PackageManager package_manager = 2; + map workspaces = 3; } message TransitiveDepsResponse { oneof response { - LockfilePackageList packages = 1; + WorkspaceDependencies dependencies = 1; string error = 2; } } @@ -79,8 +96,9 @@ message LockfilePackageList { message SubgraphRequest { bytes contents = 1; - repeated string workspaces = 2; - repeated string packages = 3; + string package_manager = 2; + repeated string workspaces = 3; + repeated string packages = 4; } message SubgraphResponse { diff --git a/crates/turborepo-ffi/src/lib.rs b/crates/turborepo-ffi/src/lib.rs index a44fca4761693..720853594463b 100644 --- a/crates/turborepo-ffi/src/lib.rs +++ b/crates/turborepo-ffi/src/lib.rs @@ -6,7 +6,7 @@ mod lockfile; use std::{mem::ManuallyDrop, path::PathBuf}; -pub use lockfile::{npm_subgraph, npm_transitive_closure}; +pub use lockfile::{npm_subgraph, transitive_closure}; mod proto { include!(concat!(env!("OUT_DIR"), "/_.rs")); diff --git a/crates/turborepo-ffi/src/lockfile.rs b/crates/turborepo-ffi/src/lockfile.rs index b02afa34734da..326b6cd11d399 100644 --- a/crates/turborepo-ffi/src/lockfile.rs +++ b/crates/turborepo-ffi/src/lockfile.rs @@ -1,8 +1,11 @@ -use thiserror::Error; -use turborepo_lockfiles::{ - npm_subgraph as real_npm_subgraph, transitive_closure, NpmLockfile, Package, +use std::{ + collections::{HashMap, HashSet}, + fmt, }; +use thiserror::Error; +use turborepo_lockfiles::{self, npm_subgraph as real_npm_subgraph, NpmLockfile, Package}; + use super::{proto, Buffer}; impl From for proto::LockfilePackage { @@ -19,16 +22,16 @@ impl From for proto::LockfilePackage { #[derive(Debug, Error)] enum Error { #[error("error performing lockfile operation")] - LockfileError(#[from] turborepo_lockfiles::Error), + Lockfile(#[from] turborepo_lockfiles::Error), #[error("error decoding protobuf")] - ProtobufError(#[from] prost::DecodeError), + Protobuf(#[from] prost::DecodeError), } #[no_mangle] -pub extern "C" fn npm_transitive_closure(buf: Buffer) -> Buffer { +pub extern "C" fn transitive_closure(buf: Buffer) -> Buffer { use proto::transitive_deps_response::Response; - let response = match npm_transitive_closure_inner(buf) { - Ok(list) => Response::Packages(list), + let response = match transitive_closure_inner(buf) { + Ok(list) => Response::Dependencies(list), Err(err) => Response::Error(err.to_string()), }; proto::TransitiveDepsResponse { @@ -37,17 +40,34 @@ pub extern "C" fn npm_transitive_closure(buf: Buffer) -> Buffer { .into() } -fn npm_transitive_closure_inner(buf: Buffer) -> Result { +fn transitive_closure_inner(buf: Buffer) -> Result { let request: proto::TransitiveDepsRequest = buf.into_proto()?; - let lockfile = NpmLockfile::load(request.contents.as_slice())?; - let transitive_deps = - transitive_closure(&lockfile, request.workspace_dir, request.unresolved_deps)?; - let list: Vec<_> = transitive_deps - .into_iter() - .map(proto::LockfilePackage::from) - .collect(); + match request.package_manager() { + proto::PackageManager::Npm => npm_transitive_closure_inner(request), + } +} - Ok(proto::LockfilePackageList { list }) +fn npm_transitive_closure_inner( + request: proto::TransitiveDepsRequest, +) -> Result { + let proto::TransitiveDepsRequest { + contents, + workspaces, + .. + } = request; + let lockfile = NpmLockfile::load(contents.as_slice())?; + let dependencies = workspaces + .into_iter() + .map(|(workspace_dir, dependencies)| { + let closure = turborepo_lockfiles::transitive_closure( + &lockfile, + &workspace_dir, + dependencies.into(), + )?; + Ok((workspace_dir, proto::LockfilePackageList::from(closure))) + }) + .collect::, Error>>()?; + Ok(proto::WorkspaceDependencies { dependencies }) } #[no_mangle] @@ -67,3 +87,32 @@ fn npm_subgraph_inner(buf: Buffer) -> Result, Error> { let contents = real_npm_subgraph(&request.contents, &request.workspaces, &request.packages)?; Ok(contents) } + +impl From for HashMap { + fn from(other: proto::PackageDependencyList) -> Self { + other + .list + .into_iter() + .map(|proto::PackageDependency { name, range }| (name, range)) + .collect() + } +} + +impl From> for proto::LockfilePackageList { + fn from(value: HashSet) -> Self { + proto::LockfilePackageList { + list: value + .into_iter() + .map(proto::LockfilePackage::from) + .collect(), + } + } +} + +impl fmt::Display for proto::PackageManager { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + proto::PackageManager::Npm => "npm", + }) + } +} diff --git a/crates/turborepo-lockfiles/src/berry/mod.rs b/crates/turborepo-lockfiles/src/berry/mod.rs index e8569313d1410..c778f0a5d02a3 100644 --- a/crates/turborepo-lockfiles/src/berry/mod.rs +++ b/crates/turborepo-lockfiles/src/berry/mod.rs @@ -753,8 +753,7 @@ mod test { .map(|(k, v)| (k.to_string(), v.to_string())) .collect(); - let closure = - transitive_closure(&lockfile, "packages/ui".to_string(), unresolved_deps).unwrap(); + let closure = transitive_closure(&lockfile, "packages/ui", unresolved_deps).unwrap(); assert!(closure.contains(&Package { key: "ajv@npm:8.11.2".into(), diff --git a/crates/turborepo-lockfiles/src/lib.rs b/crates/turborepo-lockfiles/src/lib.rs index 34d9f161e559d..74ccc86f5f939 100644 --- a/crates/turborepo-lockfiles/src/lib.rs +++ b/crates/turborepo-lockfiles/src/lib.rs @@ -36,13 +36,13 @@ pub trait Lockfile { // this should get replaced by petgraph in the future :) pub fn transitive_closure( lockfile: &L, - workspace_path: String, + workspace_path: &str, unresolved_deps: HashMap, ) -> Result, Error> { let mut transitive_deps = HashSet::new(); transitive_closure_helper( lockfile, - &workspace_path, + workspace_path, unresolved_deps, &mut transitive_deps, )?;