From 5185a13b18ccf197fd13166e928efaffa344b934 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Sat, 6 May 2023 10:13:01 -0700 Subject: [PATCH] FFI wiring --- cli/internal/ffi/bindings.h | 2 +- cli/internal/ffi/ffi.go | 21 ++ cli/internal/ffi/proto/messages.pb.go | 344 +++++++++++++++--- cli/internal/hashing/package_deps_hash.go | 100 ----- cli/internal/hashing/package_deps_hash_go.go | 112 ++++++ .../hashing/package_deps_hash_rust.go | 22 ++ crates/turborepo-ffi/messages.proto | 16 + crates/turborepo-ffi/src/lib.rs | 87 ++++- crates/turborepo-scm/src/package_deps.rs | 38 +- crates/turborepo-scm/src/status.rs | 17 +- 10 files changed, 559 insertions(+), 200 deletions(-) create mode 100644 cli/internal/hashing/package_deps_hash_go.go create mode 100644 cli/internal/hashing/package_deps_hash_rust.go diff --git a/cli/internal/ffi/bindings.h b/cli/internal/ffi/bindings.h index 381e53c6668ff..61010332dceb1 100644 --- a/cli/internal/ffi/bindings.h +++ b/cli/internal/ffi/bindings.h @@ -18,7 +18,7 @@ struct Buffer previous_content(struct Buffer buffer); struct Buffer recursive_copy(struct Buffer buffer); -struct Buffer recursive_copy(struct Buffer buffer); +struct Buffer get_package_file_hashes_from_git_index(struct Buffer buffer); struct Buffer transitive_closure(struct Buffer buf); diff --git a/cli/internal/ffi/ffi.go b/cli/internal/ffi/ffi.go index b59846fa73bb0..56c07ada7d72a 100644 --- a/cli/internal/ffi/ffi.go +++ b/cli/internal/ffi/ffi.go @@ -313,3 +313,24 @@ func GlobalChange(packageManager string, prevContents []byte, currContents []byt return resp.GetGlobalChange() } + +func GetPackageFileHashesFromGitIndex(rootPath string, packagePath string) (map[string]string, error) { + req := ffi_proto.GetPackageFileHashesFromGitIndexRequest{ + TurboRoot: rootPath, + PackagePath: packagePath, + } + reqBuf := Marshal(&req) + resBuf := C.get_package_file_hashes_from_git_index(reqBuf) + reqBuf.Free() + + resp := ffi_proto.GetPackageFileHashesFromGitIndexResponse{} + if err := Unmarshal(resBuf, resp.ProtoReflect().Interface()); err != nil { + panic(err) + } + + if err := resp.GetError(); err != "" { + return nil, errors.New(err) + } + hashes := resp.GetHashes() + return hashes.Hashes, nil +} diff --git a/cli/internal/ffi/proto/messages.pb.go b/cli/internal/ffi/proto/messages.pb.go index 553b205c07904..2232633f114bd 100644 --- a/cli/internal/ffi/proto/messages.pb.go +++ b/cli/internal/ffi/proto/messages.pb.go @@ -1662,6 +1662,190 @@ func (x *RecursiveCopyResponse) GetError() string { return "" } +type GetPackageFileHashesFromGitIndexRequest struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + TurboRoot string `protobuf:"bytes,1,opt,name=turbo_root,json=turboRoot,proto3" json:"turbo_root,omitempty"` + PackagePath string `protobuf:"bytes,2,opt,name=package_path,json=packagePath,proto3" json:"package_path,omitempty"` +} + +func (x *GetPackageFileHashesFromGitIndexRequest) Reset() { + *x = GetPackageFileHashesFromGitIndexRequest{} + if protoimpl.UnsafeEnabled { + mi := &file_turborepo_ffi_messages_proto_msgTypes[26] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetPackageFileHashesFromGitIndexRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetPackageFileHashesFromGitIndexRequest) ProtoMessage() {} + +func (x *GetPackageFileHashesFromGitIndexRequest) ProtoReflect() protoreflect.Message { + mi := &file_turborepo_ffi_messages_proto_msgTypes[26] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetPackageFileHashesFromGitIndexRequest.ProtoReflect.Descriptor instead. +func (*GetPackageFileHashesFromGitIndexRequest) Descriptor() ([]byte, []int) { + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{26} +} + +func (x *GetPackageFileHashesFromGitIndexRequest) GetTurboRoot() string { + if x != nil { + return x.TurboRoot + } + return "" +} + +func (x *GetPackageFileHashesFromGitIndexRequest) GetPackagePath() string { + if x != nil { + return x.PackagePath + } + return "" +} + +type FileHashes struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Hashes map[string]string `protobuf:"bytes,1,rep,name=hashes,proto3" json:"hashes,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"` +} + +func (x *FileHashes) Reset() { + *x = FileHashes{} + if protoimpl.UnsafeEnabled { + mi := &file_turborepo_ffi_messages_proto_msgTypes[27] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *FileHashes) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*FileHashes) ProtoMessage() {} + +func (x *FileHashes) ProtoReflect() protoreflect.Message { + mi := &file_turborepo_ffi_messages_proto_msgTypes[27] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use FileHashes.ProtoReflect.Descriptor instead. +func (*FileHashes) Descriptor() ([]byte, []int) { + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{27} +} + +func (x *FileHashes) GetHashes() map[string]string { + if x != nil { + return x.Hashes + } + return nil +} + +type GetPackageFileHashesFromGitIndexResponse struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + // Types that are assignable to Response: + // *GetPackageFileHashesFromGitIndexResponse_Hashes + // *GetPackageFileHashesFromGitIndexResponse_Error + Response isGetPackageFileHashesFromGitIndexResponse_Response `protobuf_oneof:"response"` +} + +func (x *GetPackageFileHashesFromGitIndexResponse) Reset() { + *x = GetPackageFileHashesFromGitIndexResponse{} + if protoimpl.UnsafeEnabled { + mi := &file_turborepo_ffi_messages_proto_msgTypes[28] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *GetPackageFileHashesFromGitIndexResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*GetPackageFileHashesFromGitIndexResponse) ProtoMessage() {} + +func (x *GetPackageFileHashesFromGitIndexResponse) ProtoReflect() protoreflect.Message { + mi := &file_turborepo_ffi_messages_proto_msgTypes[28] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use GetPackageFileHashesFromGitIndexResponse.ProtoReflect.Descriptor instead. +func (*GetPackageFileHashesFromGitIndexResponse) Descriptor() ([]byte, []int) { + return file_turborepo_ffi_messages_proto_rawDescGZIP(), []int{28} +} + +func (m *GetPackageFileHashesFromGitIndexResponse) GetResponse() isGetPackageFileHashesFromGitIndexResponse_Response { + if m != nil { + return m.Response + } + return nil +} + +func (x *GetPackageFileHashesFromGitIndexResponse) GetHashes() *FileHashes { + if x, ok := x.GetResponse().(*GetPackageFileHashesFromGitIndexResponse_Hashes); ok { + return x.Hashes + } + return nil +} + +func (x *GetPackageFileHashesFromGitIndexResponse) GetError() string { + if x, ok := x.GetResponse().(*GetPackageFileHashesFromGitIndexResponse_Error); ok { + return x.Error + } + return "" +} + +type isGetPackageFileHashesFromGitIndexResponse_Response interface { + isGetPackageFileHashesFromGitIndexResponse_Response() +} + +type GetPackageFileHashesFromGitIndexResponse_Hashes struct { + Hashes *FileHashes `protobuf:"bytes,1,opt,name=hashes,proto3,oneof"` +} + +type GetPackageFileHashesFromGitIndexResponse_Error struct { + Error string `protobuf:"bytes,2,opt,name=error,proto3,oneof"` +} + +func (*GetPackageFileHashesFromGitIndexResponse_Hashes) isGetPackageFileHashesFromGitIndexResponse_Response() { +} + +func (*GetPackageFileHashesFromGitIndexResponse_Error) isGetPackageFileHashesFromGitIndexResponse_Response() { +} + var File_turborepo_ffi_messages_proto protoreflect.FileDescriptor var file_turborepo_ffi_messages_proto_rawDesc = []byte{ @@ -1843,10 +2027,32 @@ var file_turborepo_ffi_messages_proto_rawDesc = []byte{ 0x76, 0x65, 0x43, 0x6f, 0x70, 0x79, 0x52, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x19, 0x0a, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x88, 0x01, 0x01, 0x42, 0x08, 0x0a, 0x06, 0x5f, 0x65, 0x72, - 0x72, 0x6f, 0x72, 0x2a, 0x24, 0x0a, 0x0e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, - 0x6e, 0x61, 0x67, 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x50, 0x4d, 0x10, 0x00, 0x12, 0x09, - 0x0a, 0x05, 0x42, 0x45, 0x52, 0x52, 0x59, 0x10, 0x01, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, 0x69, - 0x2f, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x72, 0x6f, 0x72, 0x22, 0x6b, 0x0a, 0x27, 0x47, 0x65, 0x74, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, + 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x47, + 0x69, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, 0x71, 0x75, 0x65, 0x73, 0x74, 0x12, 0x1d, + 0x0a, 0x0a, 0x74, 0x75, 0x72, 0x62, 0x6f, 0x5f, 0x72, 0x6f, 0x6f, 0x74, 0x18, 0x01, 0x20, 0x01, + 0x28, 0x09, 0x52, 0x09, 0x74, 0x75, 0x72, 0x62, 0x6f, 0x52, 0x6f, 0x6f, 0x74, 0x12, 0x21, 0x0a, + 0x0c, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x5f, 0x70, 0x61, 0x74, 0x68, 0x18, 0x02, 0x20, + 0x01, 0x28, 0x09, 0x52, 0x0b, 0x70, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x50, 0x61, 0x74, 0x68, + 0x22, 0x78, 0x0a, 0x0a, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x2f, + 0x0a, 0x06, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, 0x32, 0x17, + 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x2e, 0x48, 0x61, 0x73, 0x68, + 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x52, 0x06, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x1a, + 0x39, 0x0a, 0x0b, 0x48, 0x61, 0x73, 0x68, 0x65, 0x73, 0x45, 0x6e, 0x74, 0x72, 0x79, 0x12, 0x10, + 0x0a, 0x03, 0x6b, 0x65, 0x79, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x03, 0x6b, 0x65, 0x79, + 0x12, 0x14, 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x52, + 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x3a, 0x02, 0x38, 0x01, 0x22, 0x75, 0x0a, 0x28, 0x47, 0x65, + 0x74, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, 0x68, + 0x65, 0x73, 0x46, 0x72, 0x6f, 0x6d, 0x47, 0x69, 0x74, 0x49, 0x6e, 0x64, 0x65, 0x78, 0x52, 0x65, + 0x73, 0x70, 0x6f, 0x6e, 0x73, 0x65, 0x12, 0x25, 0x0a, 0x06, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, + 0x18, 0x01, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x0b, 0x2e, 0x46, 0x69, 0x6c, 0x65, 0x48, 0x61, 0x73, + 0x68, 0x65, 0x73, 0x48, 0x00, 0x52, 0x06, 0x68, 0x61, 0x73, 0x68, 0x65, 0x73, 0x12, 0x16, 0x0a, + 0x05, 0x65, 0x72, 0x72, 0x6f, 0x72, 0x18, 0x02, 0x20, 0x01, 0x28, 0x09, 0x48, 0x00, 0x52, 0x05, + 0x65, 0x72, 0x72, 0x6f, 0x72, 0x42, 0x0a, 0x0a, 0x08, 0x72, 0x65, 0x73, 0x70, 0x6f, 0x6e, 0x73, + 0x65, 0x2a, 0x24, 0x0a, 0x0e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x4d, 0x61, 0x6e, 0x61, + 0x67, 0x65, 0x72, 0x12, 0x07, 0x0a, 0x03, 0x4e, 0x50, 0x4d, 0x10, 0x00, 0x12, 0x09, 0x0a, 0x05, + 0x42, 0x45, 0x52, 0x52, 0x59, 0x10, 0x01, 0x42, 0x0b, 0x5a, 0x09, 0x66, 0x66, 0x69, 0x2f, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( @@ -1862,62 +2068,68 @@ func file_turborepo_ffi_messages_proto_rawDescGZIP() []byte { } var file_turborepo_ffi_messages_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_turborepo_ffi_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 29) +var file_turborepo_ffi_messages_proto_msgTypes = make([]protoimpl.MessageInfo, 33) var file_turborepo_ffi_messages_proto_goTypes = []interface{}{ - (PackageManager)(0), // 0: PackageManager - (*TurboDataDirResp)(nil), // 1: TurboDataDirResp - (*GlobReq)(nil), // 2: GlobReq - (*GlobResp)(nil), // 3: GlobResp - (*GlobRespList)(nil), // 4: GlobRespList - (*ChangedFilesReq)(nil), // 5: ChangedFilesReq - (*ChangedFilesResp)(nil), // 6: ChangedFilesResp - (*ChangedFilesList)(nil), // 7: ChangedFilesList - (*PreviousContentReq)(nil), // 8: PreviousContentReq - (*PreviousContentResp)(nil), // 9: PreviousContentResp - (*PackageDependency)(nil), // 10: PackageDependency - (*PackageDependencyList)(nil), // 11: PackageDependencyList - (*WorkspaceDependencies)(nil), // 12: WorkspaceDependencies - (*TransitiveDepsRequest)(nil), // 13: TransitiveDepsRequest - (*TransitiveDepsResponse)(nil), // 14: TransitiveDepsResponse - (*AdditionalBerryData)(nil), // 15: AdditionalBerryData - (*LockfilePackage)(nil), // 16: LockfilePackage - (*LockfilePackageList)(nil), // 17: LockfilePackageList - (*SubgraphRequest)(nil), // 18: SubgraphRequest - (*SubgraphResponse)(nil), // 19: SubgraphResponse - (*PatchesRequest)(nil), // 20: PatchesRequest - (*PatchesResponse)(nil), // 21: PatchesResponse - (*Patches)(nil), // 22: Patches - (*GlobalChangeRequest)(nil), // 23: GlobalChangeRequest - (*GlobalChangeResponse)(nil), // 24: GlobalChangeResponse - (*RecursiveCopyRequest)(nil), // 25: RecursiveCopyRequest - (*RecursiveCopyResponse)(nil), // 26: RecursiveCopyResponse - nil, // 27: WorkspaceDependencies.DependenciesEntry - nil, // 28: TransitiveDepsRequest.WorkspacesEntry - nil, // 29: AdditionalBerryData.ResolutionsEntry + (PackageManager)(0), // 0: PackageManager + (*TurboDataDirResp)(nil), // 1: TurboDataDirResp + (*GlobReq)(nil), // 2: GlobReq + (*GlobResp)(nil), // 3: GlobResp + (*GlobRespList)(nil), // 4: GlobRespList + (*ChangedFilesReq)(nil), // 5: ChangedFilesReq + (*ChangedFilesResp)(nil), // 6: ChangedFilesResp + (*ChangedFilesList)(nil), // 7: ChangedFilesList + (*PreviousContentReq)(nil), // 8: PreviousContentReq + (*PreviousContentResp)(nil), // 9: PreviousContentResp + (*PackageDependency)(nil), // 10: PackageDependency + (*PackageDependencyList)(nil), // 11: PackageDependencyList + (*WorkspaceDependencies)(nil), // 12: WorkspaceDependencies + (*TransitiveDepsRequest)(nil), // 13: TransitiveDepsRequest + (*TransitiveDepsResponse)(nil), // 14: TransitiveDepsResponse + (*AdditionalBerryData)(nil), // 15: AdditionalBerryData + (*LockfilePackage)(nil), // 16: LockfilePackage + (*LockfilePackageList)(nil), // 17: LockfilePackageList + (*SubgraphRequest)(nil), // 18: SubgraphRequest + (*SubgraphResponse)(nil), // 19: SubgraphResponse + (*PatchesRequest)(nil), // 20: PatchesRequest + (*PatchesResponse)(nil), // 21: PatchesResponse + (*Patches)(nil), // 22: Patches + (*GlobalChangeRequest)(nil), // 23: GlobalChangeRequest + (*GlobalChangeResponse)(nil), // 24: GlobalChangeResponse + (*RecursiveCopyRequest)(nil), // 25: RecursiveCopyRequest + (*RecursiveCopyResponse)(nil), // 26: RecursiveCopyResponse + (*GetPackageFileHashesFromGitIndexRequest)(nil), // 27: GetPackageFileHashesFromGitIndexRequest + (*FileHashes)(nil), // 28: FileHashes + (*GetPackageFileHashesFromGitIndexResponse)(nil), // 29: GetPackageFileHashesFromGitIndexResponse + nil, // 30: WorkspaceDependencies.DependenciesEntry + nil, // 31: TransitiveDepsRequest.WorkspacesEntry + nil, // 32: AdditionalBerryData.ResolutionsEntry + nil, // 33: FileHashes.HashesEntry } var file_turborepo_ffi_messages_proto_depIdxs = []int32{ 4, // 0: GlobResp.files:type_name -> GlobRespList 7, // 1: ChangedFilesResp.files:type_name -> ChangedFilesList 10, // 2: PackageDependencyList.list:type_name -> PackageDependency - 27, // 3: WorkspaceDependencies.dependencies:type_name -> WorkspaceDependencies.DependenciesEntry + 30, // 3: WorkspaceDependencies.dependencies:type_name -> WorkspaceDependencies.DependenciesEntry 0, // 4: TransitiveDepsRequest.package_manager:type_name -> PackageManager - 28, // 5: TransitiveDepsRequest.workspaces:type_name -> TransitiveDepsRequest.WorkspacesEntry + 31, // 5: TransitiveDepsRequest.workspaces:type_name -> TransitiveDepsRequest.WorkspacesEntry 15, // 6: TransitiveDepsRequest.resolutions:type_name -> AdditionalBerryData 12, // 7: TransitiveDepsResponse.dependencies:type_name -> WorkspaceDependencies - 29, // 8: AdditionalBerryData.resolutions:type_name -> AdditionalBerryData.ResolutionsEntry + 32, // 8: AdditionalBerryData.resolutions:type_name -> AdditionalBerryData.ResolutionsEntry 16, // 9: LockfilePackageList.list:type_name -> LockfilePackage 0, // 10: SubgraphRequest.package_manager:type_name -> PackageManager 15, // 11: SubgraphRequest.resolutions:type_name -> AdditionalBerryData 0, // 12: PatchesRequest.package_manager:type_name -> PackageManager 22, // 13: PatchesResponse.patches:type_name -> Patches 0, // 14: GlobalChangeRequest.package_manager:type_name -> PackageManager - 17, // 15: WorkspaceDependencies.DependenciesEntry.value:type_name -> LockfilePackageList - 11, // 16: TransitiveDepsRequest.WorkspacesEntry.value:type_name -> PackageDependencyList - 17, // [17:17] is the sub-list for method output_type - 17, // [17:17] is the sub-list for method input_type - 17, // [17:17] is the sub-list for extension type_name - 17, // [17:17] is the sub-list for extension extendee - 0, // [0:17] is the sub-list for field type_name + 33, // 15: FileHashes.hashes:type_name -> FileHashes.HashesEntry + 28, // 16: GetPackageFileHashesFromGitIndexResponse.hashes:type_name -> FileHashes + 17, // 17: WorkspaceDependencies.DependenciesEntry.value:type_name -> LockfilePackageList + 11, // 18: TransitiveDepsRequest.WorkspacesEntry.value:type_name -> PackageDependencyList + 19, // [19:19] is the sub-list for method output_type + 19, // [19:19] is the sub-list for method input_type + 19, // [19:19] is the sub-list for extension type_name + 19, // [19:19] is the sub-list for extension extendee + 0, // [0:19] is the sub-list for field type_name } func init() { file_turborepo_ffi_messages_proto_init() } @@ -2238,6 +2450,42 @@ func file_turborepo_ffi_messages_proto_init() { return nil } } + file_turborepo_ffi_messages_proto_msgTypes[26].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetPackageFileHashesFromGitIndexRequest); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_turborepo_ffi_messages_proto_msgTypes[27].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*FileHashes); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_turborepo_ffi_messages_proto_msgTypes[28].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*GetPackageFileHashesFromGitIndexResponse); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } } file_turborepo_ffi_messages_proto_msgTypes[2].OneofWrappers = []interface{}{ (*GlobResp_Files)(nil), @@ -2267,13 +2515,17 @@ func file_turborepo_ffi_messages_proto_init() { (*PatchesResponse_Error)(nil), } file_turborepo_ffi_messages_proto_msgTypes[25].OneofWrappers = []interface{}{} + file_turborepo_ffi_messages_proto_msgTypes[28].OneofWrappers = []interface{}{ + (*GetPackageFileHashesFromGitIndexResponse_Hashes)(nil), + (*GetPackageFileHashesFromGitIndexResponse_Error)(nil), + } type x struct{} out := protoimpl.TypeBuilder{ File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: file_turborepo_ffi_messages_proto_rawDesc, NumEnums: 1, - NumMessages: 29, + NumMessages: 33, NumExtensions: 0, NumServices: 0, }, diff --git a/cli/internal/hashing/package_deps_hash.go b/cli/internal/hashing/package_deps_hash.go index c51c056b9dcf7..dfa972585729d 100644 --- a/cli/internal/hashing/package_deps_hash.go +++ b/cli/internal/hashing/package_deps_hash.go @@ -28,48 +28,6 @@ type PackageDepsOptions struct { InputPatterns []string } -func getPackageFileHashesFromGitIndex(rootPath turbopath.AbsoluteSystemPath, packagePath turbopath.AnchoredSystemPath) (map[turbopath.AnchoredUnixPath]string, error) { - var result map[turbopath.AnchoredUnixPath]string - absolutePackagePath := packagePath.RestoreAnchor(rootPath) - - // Get the state of the git index. - gitLsTreeOutput, err := gitLsTree(absolutePackagePath) - if err != nil { - return nil, fmt.Errorf("could not get git hashes for files in package %s: %w", packagePath, err) - } - result = gitLsTreeOutput - - // Update the with the state of the working directory. - // The paths returned from this call are anchored at the package directory - gitStatusOutput, err := gitStatus(absolutePackagePath) - if err != nil { - return nil, fmt.Errorf("Could not get git hashes from git status: %v", err) - } - - // Review status output to identify the delta. - var filesToHash []turbopath.AnchoredSystemPath - for filePath, status := range gitStatusOutput { - if status.isDelete() { - delete(result, filePath) - } else { - filesToHash = append(filesToHash, filePath.ToSystemPath()) - } - } - - // Get the hashes for any modified files in the working directory. - hashes, err := GetHashesForFiles(absolutePackagePath, filesToHash) - if err != nil { - return nil, err - } - - // Zip up file paths and hashes together - for filePath, hash := range hashes { - result[filePath] = hash - } - - return result, nil -} - func safeCompileIgnoreFile(filepath turbopath.AbsoluteSystemPath) (*gitignore.GitIgnore, error) { if filepath.FileExists() { return gitignore.CompileIgnoreFile(filepath.ToString()) @@ -496,61 +454,3 @@ type statusCode struct { func (s statusCode) isDelete() bool { return s.x == "D" || s.y == "D" } - -// gitStatus returns a map of paths to their `git` status code. This can be used to identify what should -// be done with files that do not currently match what is in the index. -// -// Note: `git status -z`'s relative path results are relative to the repository's location. -// We need to calculate where the repository's location is in order to determine what the full path is -// before we can return those paths relative to the calling directory, normalizing to the behavior of -// `ls-files` and `ls-tree`. -func gitStatus(rootPath turbopath.AbsoluteSystemPath) (map[turbopath.AnchoredUnixPath]statusCode, error) { - cmd := exec.Command( - "git", // Using `git` from $PATH, - "status", // tell me about the status of the working tree, - "--untracked-files", // including information about untracked files, - "--no-renames", // do not detect renames, - "-z", // with each file path relative to the repository root and \000-terminated, - "--", // and any additional argument you see is a path, promise. - ) - cmd.Args = append(cmd.Args, ".") // Operate in the current directory instead of the root of the working tree. - cmd.Dir = rootPath.ToString() // Include files only from this directory. - - entries, err := runGitCommand(cmd, "status", gitoutput.NewStatusReader) - if err != nil { - return nil, err - } - - output := make(map[turbopath.AnchoredUnixPath]statusCode, len(entries)) - convertedRootPath := turbopath.AbsoluteSystemPathFromUpstream(rootPath.ToString()) - - traversePath, err := memoizedGetTraversePath(convertedRootPath) - if err != nil { - return nil, err - } - - for _, entry := range entries { - statusEntry := gitoutput.StatusEntry(entry) - // Anchored at repository. - pathFromStatus := turbopath.AnchoredUnixPathFromUpstream(statusEntry.GetField(gitoutput.Path)) - var outputPath turbopath.AnchoredUnixPath - - if len(traversePath) > 0 { - repositoryPath := convertedRootPath.Join(traversePath.ToSystemPath()) - fileFullPath := pathFromStatus.ToSystemPath().RestoreAnchor(repositoryPath) - - relativePath, err := fileFullPath.RelativeTo(convertedRootPath) - if err != nil { - return nil, err - } - - outputPath = relativePath.ToUnixPath() - } else { - outputPath = pathFromStatus - } - - output[outputPath] = statusCode{x: statusEntry.GetField(gitoutput.StatusX), y: statusEntry.GetField(gitoutput.StatusY)} - } - - return output, nil -} diff --git a/cli/internal/hashing/package_deps_hash_go.go b/cli/internal/hashing/package_deps_hash_go.go new file mode 100644 index 0000000000000..46e5db6a65774 --- /dev/null +++ b/cli/internal/hashing/package_deps_hash_go.go @@ -0,0 +1,112 @@ +//go:build go || !rust +// +build go !rust + +package hashing + +import ( + "fmt" + "os/exec" + + "github.com/vercel/turbo/cli/internal/encoding/gitoutput" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +func getPackageFileHashesFromGitIndex(rootPath turbopath.AbsoluteSystemPath, packagePath turbopath.AnchoredSystemPath) (map[turbopath.AnchoredUnixPath]string, error) { + var result map[turbopath.AnchoredUnixPath]string + absolutePackagePath := packagePath.RestoreAnchor(rootPath) + + // Get the state of the git index. + gitLsTreeOutput, err := gitLsTree(absolutePackagePath) + if err != nil { + return nil, fmt.Errorf("could not get git hashes for files in package %s: %w", packagePath, err) + } + result = gitLsTreeOutput + + // Update the with the state of the working directory. + // The paths returned from this call are anchored at the package directory + gitStatusOutput, err := gitStatus(absolutePackagePath) + if err != nil { + return nil, fmt.Errorf("Could not get git hashes from git status: %v", err) + } + + // Review status output to identify the delta. + var filesToHash []turbopath.AnchoredSystemPath + for filePath, status := range gitStatusOutput { + if status.isDelete() { + delete(result, filePath) + } else { + filesToHash = append(filesToHash, filePath.ToSystemPath()) + } + } + + // Get the hashes for any modified files in the working directory. + hashes, err := GetHashesForFiles(absolutePackagePath, filesToHash) + if err != nil { + return nil, err + } + + // Zip up file paths and hashes together + for filePath, hash := range hashes { + result[filePath] = hash + } + + return result, nil +} + +// gitStatus returns a map of paths to their `git` status code. This can be used to identify what should +// be done with files that do not currently match what is in the index. +// +// Note: `git status -z`'s relative path results are relative to the repository's location. +// We need to calculate where the repository's location is in order to determine what the full path is +// before we can return those paths relative to the calling directory, normalizing to the behavior of +// `ls-files` and `ls-tree`. +func gitStatus(rootPath turbopath.AbsoluteSystemPath) (map[turbopath.AnchoredUnixPath]statusCode, error) { + cmd := exec.Command( + "git", // Using `git` from $PATH, + "status", // tell me about the status of the working tree, + "--untracked-files", // including information about untracked files, + "--no-renames", // do not detect renames, + "-z", // with each file path relative to the repository root and \000-terminated, + "--", // and any additional argument you see is a path, promise. + ) + cmd.Args = append(cmd.Args, ".") // Operate in the current directory instead of the root of the working tree. + cmd.Dir = rootPath.ToString() // Include files only from this directory. + + entries, err := runGitCommand(cmd, "status", gitoutput.NewStatusReader) + if err != nil { + return nil, err + } + + output := make(map[turbopath.AnchoredUnixPath]statusCode, len(entries)) + convertedRootPath := turbopath.AbsoluteSystemPathFromUpstream(rootPath.ToString()) + + traversePath, err := memoizedGetTraversePath(convertedRootPath) + if err != nil { + return nil, err + } + + for _, entry := range entries { + statusEntry := gitoutput.StatusEntry(entry) + // Anchored at repository. + pathFromStatus := turbopath.AnchoredUnixPathFromUpstream(statusEntry.GetField(gitoutput.Path)) + var outputPath turbopath.AnchoredUnixPath + + if len(traversePath) > 0 { + repositoryPath := convertedRootPath.Join(traversePath.ToSystemPath()) + fileFullPath := pathFromStatus.ToSystemPath().RestoreAnchor(repositoryPath) + + relativePath, err := fileFullPath.RelativeTo(convertedRootPath) + if err != nil { + return nil, err + } + + outputPath = relativePath.ToUnixPath() + } else { + outputPath = pathFromStatus + } + + output[outputPath] = statusCode{x: statusEntry.GetField(gitoutput.StatusX), y: statusEntry.GetField(gitoutput.StatusY)} + } + + return output, nil +} diff --git a/cli/internal/hashing/package_deps_hash_rust.go b/cli/internal/hashing/package_deps_hash_rust.go new file mode 100644 index 0000000000000..4f5aa1dd13ae9 --- /dev/null +++ b/cli/internal/hashing/package_deps_hash_rust.go @@ -0,0 +1,22 @@ +//go:build rust +// +build rust + +package hashing + +import ( + "github.com/vercel/turbo/cli/internal/ffi" + "github.com/vercel/turbo/cli/internal/turbopath" +) + +func getPackageFileHashesFromGitIndex(rootPath turbopath.AbsoluteSystemPath, packagePath turbopath.AnchoredSystemPath) (map[turbopath.AnchoredUnixPath]string, error) { + rawHashes, err := ffi.GetPackageFileHashesFromGitIndex(rootPath.ToString(), packagePath.ToString()) + if err != nil { + return nil, err + } + + hashes := make(map[turbopath.AnchoredUnixPath]string, len(rawHashes)) + for rawPath, hash := range rawHashes { + hashes[turbopath.AnchoredUnixPathFromUpstream(rawPath)] = hash + } + return hashes, nil +} diff --git a/crates/turborepo-ffi/messages.proto b/crates/turborepo-ffi/messages.proto index ce7b189a1f3a2..d4408d21daf5f 100644 --- a/crates/turborepo-ffi/messages.proto +++ b/crates/turborepo-ffi/messages.proto @@ -149,3 +149,19 @@ message RecursiveCopyRequest { message RecursiveCopyResponse { optional string error = 1; } + +message GetPackageFileHashesFromGitIndexRequest { + string turbo_root = 1; + string package_path = 2; +} + +message FileHashes { + map hashes = 1; +} + +message GetPackageFileHashesFromGitIndexResponse { + oneof response { + FileHashes hashes = 1; + string error = 2; + } +} diff --git a/crates/turborepo-ffi/src/lib.rs b/crates/turborepo-ffi/src/lib.rs index 9e37736159b65..52e10ff72fd02 100644 --- a/crates/turborepo-ffi/src/lib.rs +++ b/crates/turborepo-ffi/src/lib.rs @@ -4,10 +4,10 @@ //! and in ffi.go before modifying this file. mod lockfile; -use std::{mem::ManuallyDrop, path::PathBuf}; +use std::{collections::HashMap, mem::ManuallyDrop, path::PathBuf}; pub use lockfile::{patches, subgraph, transitive_closure}; -use turbopath::AbsoluteSystemPathBuf; +use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPathBuf}; mod proto { include!(concat!(env!("OUT_DIR"), "/_.rs")); @@ -165,24 +165,85 @@ pub extern "C" fn recursive_copy(buffer: Buffer) -> Buffer { } #[no_mangle] -pub extern "C" fn recursive_copy(buffer: Buffer) -> Buffer { - let req: proto::RecursiveCopyRequest = match buffer.into_proto() { +pub extern "C" fn get_package_file_hashes_from_git_index(buffer: Buffer) -> Buffer { + let req: proto::GetPackageFileHashesFromGitIndexRequest = match buffer.into_proto() { Ok(req) => req, Err(err) => { - let resp = proto::RecursiveCopyResponse { - error: Some(err.to_string()), + let resp = proto::GetPackageFileHashesFromGitIndexResponse { + response: Some( + proto::get_package_file_hashes_from_git_index_response::Response::Error( + err.to_string(), + ), + ), }; return resp.into(); } }; - let response = match turborepo_fs::recursive_copy( - &AbsoluteSystemPathBuf::new_unchecked(req.src), - &AbsoluteSystemPathBuf::new_unchecked(req.dst), + + let turbo_root = match AbsoluteSystemPathBuf::new(req.turbo_root) { + Ok(turbo_root) => turbo_root, + Err(err) => { + let resp = proto::GetPackageFileHashesFromGitIndexResponse { + response: Some( + proto::get_package_file_hashes_from_git_index_response::Response::Error( + err.to_string(), + ), + ), + }; + return resp.into(); + } + }; + let package_path = match AnchoredSystemPathBuf::from_raw(req.package_path) { + Ok(package_path) => package_path, + Err(err) => { + let resp = proto::GetPackageFileHashesFromGitIndexResponse { + response: Some( + proto::get_package_file_hashes_from_git_index_response::Response::Error( + err.to_string(), + ), + ), + }; + return resp.into(); + } + }; + let response = match turborepo_scm::package_deps::get_package_file_hashes_from_git_index( + &turbo_root, + &package_path, ) { - Ok(()) => proto::RecursiveCopyResponse { error: None }, - Err(e) => proto::RecursiveCopyResponse { - error: Some(e.to_string()), - }, + Ok(hashes) => { + let mut to_return = HashMap::new(); + for (filename, hash) in hashes { + let filename = match filename.as_str() { + Ok(s) => s.to_owned(), + Err(err) => { + let resp = proto::GetPackageFileHashesFromGitIndexResponse { + response: Some(proto::get_package_file_hashes_from_git_index_response::Response::Error(err.to_string())) + }; + return resp.into(); + } + }; + to_return.insert(filename, hash); + } + let file_hashes = proto::FileHashes { hashes: to_return }; + let resp = proto::GetPackageFileHashesFromGitIndexResponse { + response: Some( + proto::get_package_file_hashes_from_git_index_response::Response::Hashes( + file_hashes, + ), + ), + }; + resp + } + Err(err) => { + let resp = proto::GetPackageFileHashesFromGitIndexResponse { + response: Some( + proto::get_package_file_hashes_from_git_index_response::Response::Error( + err.to_string(), + ), + ), + }; + return resp.into(); + } }; response.into() } diff --git a/crates/turborepo-scm/src/package_deps.rs b/crates/turborepo-scm/src/package_deps.rs index db0d85fef95f1..ffb26ed4ea741 100644 --- a/crates/turborepo-scm/src/package_deps.rs +++ b/crates/turborepo-scm/src/package_deps.rs @@ -1,50 +1,26 @@ use std::{collections::HashMap, process::Command}; -use anyhow::{anyhow, Result}; +use anyhow::Result; use turbopath::{AbsoluteSystemPathBuf, AnchoredSystemPathBuf, RelativeUnixPathBuf}; use crate::{hash_object::hash_objects, ls_tree::git_ls_tree, status::append_git_status}; pub type GitHashes = HashMap; -pub fn get_package_deps( +pub fn get_package_file_hashes_from_git_index( turbo_root: &AbsoluteSystemPathBuf, package_path: &AnchoredSystemPathBuf, - inputs: &[&str], ) -> Result { // TODO: memoize git root -> turbo root calculation once we aren't crossing ffi let git_root = find_git_root(turbo_root)?; let full_pkg_path = turbo_root.resolve(package_path); let git_to_pkg_path = git_root.anchor(&full_pkg_path)?; let pkg_prefix = git_to_pkg_path.to_unix()?; - let result = if inputs.len() == 0 { - let mut hashes = git_ls_tree(&full_pkg_path)?; - // Note: to_hash is *git repo relative* - let to_hash = append_git_status(&full_pkg_path, &pkg_prefix, inputs, &mut hashes)?; - hash_objects(&full_pkg_path, to_hash, &pkg_prefix, &mut hashes)?; - hashes - } else { - let pkg_prefix_str = pkg_prefix.as_str()?; - let mut inputs = inputs.to_vec(); - inputs.push("package.json"); - inputs.push("turbo.json"); - let mut prefixed_input_patterns = vec![]; - let mut prefixed_exclude_patterns = vec![]; - for input in inputs { - if input.starts_with("!") { - let glob = input - .get(1..) - .ok_or_else(|| anyhow!("invalid glob: {}", input))?; - let pkg_glob = format!("{}/{}", pkg_prefix_str, glob); - prefixed_exclude_patterns.push(pkg_glob); - } else { - let pkg_glob = format!("{}/{}", pkg_prefix_str, input); - prefixed_input_patterns.push(pkg_glob); - } - } - let files_to_hash = unimplemented!(); - }; - Ok(result) + let mut hashes = git_ls_tree(&full_pkg_path)?; + // Note: to_hash is *git repo relative* + let to_hash = append_git_status(&full_pkg_path, &pkg_prefix, &mut hashes)?; + hash_objects(&full_pkg_path, to_hash, &pkg_prefix, &mut hashes)?; + Ok(hashes) } pub(crate) fn find_git_root(turbo_root: &AbsoluteSystemPathBuf) -> Result { diff --git a/crates/turborepo-scm/src/status.rs b/crates/turborepo-scm/src/status.rs index 7d65e8d2aa49e..0111e57f77a14 100644 --- a/crates/turborepo-scm/src/status.rs +++ b/crates/turborepo-scm/src/status.rs @@ -13,18 +13,17 @@ use crate::{package_deps::GitHashes, Error}; pub(crate) fn append_git_status( root_path: &AbsoluteSystemPathBuf, pkg_prefix: &RelativeUnixPathBuf, - patterns: &[&str], hashes: &mut GitHashes, ) -> Result> { - let mut args = vec!["status", "--untracked-files", "--no-renames", "-z", "--"]; - if patterns.len() == 0 { - args.push("."); - } else { - let mut patterns = Vec::from(patterns); - args.append(&mut patterns); - } let mut git = Command::new("git") - .args(args.as_slice()) + .args(&[ + "status", + "--untracked-files", + "--no-renames", + "-z", + "--", + ".", + ]) .current_dir(root_path) .stdout(Stdio::piped()) .stderr(Stdio::piped())