New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Prefactor package hashing #4816
Merged
kodiakhq
merged 14 commits into
vercel:main
from
nathanhammond:prefactor-package-hashing
May 4, 2023
Merged
Changes from 1 commit
Commits
Show all changes
14 commits
Select commit
Hold shift + click to select a range
d835690
Split out getPackageFileHashesFromGitIndex.
c760457
Split out getPackageFileHashesFromInputs.
4720f40
Split out GetPackageFileHashesFromProcessingGitIgnore.
4b1fd4f
Move test.
be50d8e
Move the fallback behavior inline.
2b7e154
Make linter happy.
7bb65aa
Remove unused function.
57cd58e
BUGFIX: use turbopath, correct path error.
8258d86
Force hashes through a helper function that handles fallback.
a55c004
Stop directly calling gitHashObject
a5f3b1b
Force getting hashes for files through a single code path.
5f3bcbe
BUGFIX: manual fallback now propagates error.
269e49b
Remove packageFileSpec. Key needs to be shareable.
2df5983
Make linter happy.
File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -26,6 +26,53 @@ type PackageDepsOptions struct { | |
InputPatterns []string | ||
} | ||
|
||
func getPackageFileHashesFromGitIndex(rootPath turbopath.AbsoluteSystemPath, packagePath turbopath.AnchoredSystemPath) (map[turbopath.AnchoredUnixPath]string, error) { | ||
var result map[turbopath.AnchoredUnixPath]string | ||
absolutePackagePath := packagePath.RestoreAnchor(rootPath) | ||
|
||
// Get the state of the git index. | ||
gitLsTreeOutput, err := gitLsTree(absolutePackagePath) | ||
if err != nil { | ||
return nil, fmt.Errorf("could not get git hashes for files in package %s: %w", packagePath, err) | ||
} | ||
result = gitLsTreeOutput | ||
|
||
// Update the with the state of the working directory. | ||
// The paths returned from this call are anchored at the package directory | ||
gitStatusOutput, err := gitStatus(absolutePackagePath) | ||
if err != nil { | ||
return nil, fmt.Errorf("Could not get git hashes from git status: %v", err) | ||
} | ||
|
||
// Review status output to identify the delta. | ||
var filesToHash []turbopath.AnchoredSystemPath | ||
for filePath, status := range gitStatusOutput { | ||
if status.isDelete() { | ||
delete(result, filePath) | ||
} else { | ||
filesToHash = append(filesToHash, filePath.ToSystemPath()) | ||
} | ||
} | ||
|
||
// Ask `git` to hash any files currently in the working directory. | ||
hashes, err := gitHashObject(absolutePackagePath, filesToHash) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
// Zip up file paths and hashes together | ||
for filePath, hash := range hashes { | ||
result[filePath] = hash | ||
} | ||
|
||
return result, nil | ||
} | ||
|
||
// func getFileHashesFromProcessingGitIgnore(rootPath turbopath.AbsoluteSystemPath, packagePath turbopath.AnchoredSystemPath, inputs []string) (map[turbopath.AnchoredUnixPath]string, error) { | ||
// } | ||
// func getFileHashesFromInputs(rootPath turbopath.AbsoluteSystemPath, packagePath turbopath.AnchoredSystemPath, inputs []string) (map[turbopath.AnchoredUnixPath]string, error) { | ||
// } | ||
|
||
// GetPackageDeps Builds an object containing git hashes for the files under the specified `packagePath` folder. | ||
func GetPackageDeps(rootPath turbopath.AbsoluteSystemPath, p *PackageDepsOptions) (map[turbopath.AnchoredUnixPath]string, error) { | ||
pkgPath := rootPath.UntypedJoin(p.PackagePath.ToStringDuringMigration()) | ||
|
@@ -37,37 +84,7 @@ func GetPackageDeps(rootPath turbopath.AbsoluteSystemPath, p *PackageDepsOptions | |
copy(calculatedInputs, p.InputPatterns) | ||
|
||
if len(calculatedInputs) == 0 { | ||
gitLsTreeOutput, err := gitLsTree(pkgPath) | ||
if err != nil { | ||
return nil, fmt.Errorf("could not get git hashes for files in package %s: %w", p.PackagePath, err) | ||
} | ||
result = gitLsTreeOutput | ||
|
||
// Update the checked in hashes with the current repo status | ||
// The paths returned from this call are anchored at the package directory | ||
gitStatusOutput, err := gitStatus(pkgPath, calculatedInputs) | ||
if err != nil { | ||
return nil, fmt.Errorf("Could not get git hashes from git status: %v", err) | ||
} | ||
|
||
var filesToHash []turbopath.AnchoredSystemPath | ||
for filePath, status := range gitStatusOutput { | ||
if status.isDelete() { | ||
delete(result, filePath) | ||
} else { | ||
filesToHash = append(filesToHash, filePath.ToSystemPath()) | ||
} | ||
} | ||
|
||
hashes, err := gitHashObject(turbopath.AbsoluteSystemPathFromUpstream(pkgPath.ToString()), filesToHash) | ||
if err != nil { | ||
return nil, err | ||
} | ||
|
||
// Zip up file paths and hashes together | ||
for filePath, hash := range hashes { | ||
result[filePath] = hash | ||
} | ||
return getPackageFileHashesFromGitIndex(rootPath, p.PackagePath) | ||
} else { | ||
// Add in package.json and turbo.json to input patterns. Both file paths are relative to pkgPath | ||
// | ||
|
@@ -404,7 +421,7 @@ func (s statusCode) isDelete() bool { | |
// We need to calculate where the repository's location is in order to determine what the full path is | ||
// before we can return those paths relative to the calling directory, normalizing to the behavior of | ||
// `ls-files` and `ls-tree`. | ||
func gitStatus(rootPath turbopath.AbsoluteSystemPath, patterns []string) (map[turbopath.AnchoredUnixPath]statusCode, error) { | ||
func gitStatus(rootPath turbopath.AbsoluteSystemPath) (map[turbopath.AnchoredUnixPath]statusCode, error) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I discovered that our current use does not pass any patterns to |
||
cmd := exec.Command( | ||
"git", // Using `git` from $PATH, | ||
"status", // tell me about the status of the working tree, | ||
|
@@ -413,13 +430,8 @@ func gitStatus(rootPath turbopath.AbsoluteSystemPath, patterns []string) (map[tu | |
"-z", // with each file path relative to the repository root and \000-terminated, | ||
"--", // and any additional argument you see is a path, promise. | ||
) | ||
if len(patterns) == 0 { | ||
cmd.Args = append(cmd.Args, ".") // Operate in the current directory instead of the root of the working tree. | ||
} else { | ||
// FIXME: Globbing is using `git`'s globbing rules which are not consistent with `doublestar``. | ||
cmd.Args = append(cmd.Args, patterns...) // Pass in input patterns as arguments. | ||
} | ||
cmd.Dir = rootPath.ToString() // Include files only from this directory. | ||
cmd.Args = append(cmd.Args, ".") // Operate in the current directory instead of the root of the working tree. | ||
cmd.Dir = rootPath.ToString() // Include files only from this directory. | ||
|
||
entries, err := runGitCommand(cmd, "status", gitoutput.NewStatusReader) | ||
if err != nil { | ||
|
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Moved the body of this branch.