Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

port(turborepo): Package Manager Inference #4655

Merged
merged 8 commits into from May 3, 2023
Merged

port(turborepo): Package Manager Inference #4655

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
5ebba59
Added package manager and root package json to execution state
NicholasLYang Apr 11, 2023
f221930
Adding tests for package manager
NicholasLYang Apr 27, 2023
4efc0e4
Hooked up package manager to Go code. Added testing
NicholasLYang Apr 27, 2023
86a0517
PR feedback: Refactored into package manager detector structs
NicholasLYang May 2, 2023
caec282
Fixing up
NicholasLYang May 2, 2023
a8cc6c7
Ported more tests
NicholasLYang May 2, 2023
78ba95c
Tests and cleaing up
NicholasLYang May 2, 2023
4dec614
Update crates/turborepo-lib/src/package_manager/yarn.rs
NicholasLYang May 3, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
42 changes: 40 additions & 2 deletions Cargo.lock
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions cli/internal/context/context.go
View file
Open in desktop
Expand Up @@ -142,7 +142,7 @@ func isWorkspaceReference(packageVersion string, dependencyVersion string, cwd s
}

// SinglePackageGraph constructs a Context instance from a single package.
func SinglePackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *fs.PackageJSON) (*Context, error) {
func SinglePackageGraph(rootPackageJSON *fs.PackageJSON, packageManagerName string) (*Context, error) {
workspaceInfos := workspace.Catalog{
PackageJSONs: map[string]*fs.PackageJSON{util.RootPkgName: rootPackageJSON},
TurboConfigs: map[string]*fs.TurboJSON{},
Expand All @@ -152,7 +152,7 @@ func SinglePackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *
RootNode: core.ROOT_NODE_NAME,
}
c.WorkspaceGraph.Connect(dag.BasicEdge(util.RootPkgName, core.ROOT_NODE_NAME))
packageManager, err := packagemanager.GetPackageManager(repoRoot, rootPackageJSON)
packageManager, err := packagemanager.GetPackageManager(packageManagerName)
if err != nil {
return nil, err
}
Expand All @@ -161,7 +161,7 @@ func SinglePackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *
}

// BuildPackageGraph constructs a Context instance with information about the package dependency graph
func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *fs.PackageJSON) (*Context, error) {
func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *fs.PackageJSON, packageManagerName string) (*Context, error) {
c := &Context{}
rootpath := repoRoot.ToStringDuringMigration()
c.WorkspaceInfos = workspace.Catalog{
Expand All @@ -172,7 +172,7 @@ func BuildPackageGraph(repoRoot turbopath.AbsoluteSystemPath, rootPackageJSON *f

var warnings Warnings

packageManager, err := packagemanager.GetPackageManager(repoRoot, rootPackageJSON)
packageManager, err := packagemanager.GetPackageManager(packageManagerName)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions cli/internal/context/context_test.go
View file
Open in desktop
Expand Up @@ -141,7 +141,7 @@ func TestBuildPackageGraph_DuplicateNames(t *testing.T) {
PackageManager: "pnpm@7.15.0",
}

_, actualErr := BuildPackageGraph(path, pkgJSON)
_, actualErr := BuildPackageGraph(path, pkgJSON, "pnpm")

// Not asserting the full error message, because it includes a path with slashes and backslashes
// getting the regex incantation to check that is not worth it.
Expand All @@ -157,7 +157,7 @@ func Test_populateExternalDeps_NoTransitiveDepsWithoutLockfile(t *testing.T) {
PackageManager: "pnpm@7.15.0",
}

pm, err := packagemanager.GetPackageManager(path, pkgJSON)
pm, err := packagemanager.GetPackageManager("pnpm")
assert.NilError(t, err)
pm.UnmarshalLockfile = func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) {
return nil, errors.New("bad lockfile")
Expand Down
64 changes: 0 additions & 64 deletions cli/internal/packagemanager/berry.go
View file
Open in desktop
Expand Up @@ -2,10 +2,8 @@ package packagemanager

import (
"fmt"
"os/exec"
"strings"

"github.com/Masterminds/semver"
"github.com/pkg/errors"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/lockfile"
Expand Down Expand Up @@ -51,68 +49,6 @@ var nodejsBerry = PackageManager{
return true, nil
},

// Versions newer than 2.0 are berry, and before that we simply call them yarn.
Matches: func(manager string, version string) (bool, error) {
if manager != "yarn" {
return false, nil
}

v, err := semver.NewVersion(version)
if err != nil {
return false, fmt.Errorf("could not parse yarn version: %w", err)
}
// -0 allows pre-releases versions to be considered valid
c, err := semver.NewConstraint(">=2.0.0-0")
if err != nil {
return false, fmt.Errorf("could not create constraint: %w", err)
}

return c.Check(v), nil
},

// Detect for berry needs to identify which version of yarn is running on the system.
// Further, berry can be configured in an incompatible way, so we check for compatibility here as well.
detect: func(projectDirectory turbopath.AbsoluteSystemPath, packageManager *PackageManager) (bool, error) {
specfileExists := projectDirectory.UntypedJoin(packageManager.Specfile).FileExists()
lockfileExists := projectDirectory.UntypedJoin(packageManager.Lockfile).FileExists()

// Short-circuit, definitely not Yarn.
if !specfileExists || !lockfileExists {
return false, nil
}

cmd := exec.Command("yarn", "--version")
cmd.Dir = projectDirectory.ToString()
out, err := cmd.Output()
if err != nil {
return false, fmt.Errorf("could not detect yarn version: %w", err)
}

// See if we're a match when we compare these two things.
matches, _ := packageManager.Matches(packageManager.Slug, string(out))

// Short-circuit, definitely not Berry because version number says we're Yarn.
if !matches {
return false, nil
}

// We're Berry!

// Check for supported configuration.
isNMLinker, err := util.IsNMLinker(projectDirectory.ToStringDuringMigration())

if err != nil {
// Failed to read the linker state, so we treat an unknown configuration as a failure.
return false, fmt.Errorf("could not check if yarn is using nm-linker: %w", err)
} else if !isNMLinker {
// Not using nm-linker, so unsupported configuration.
return false, fmt.Errorf("only yarn nm-linker is supported")
}

// Berry, supported configuration.
return true, nil
},

UnmarshalLockfile: func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error) {
var resolutions map[string]string
if untypedResolutions, ok := rootPackageJSON.RawJSON["resolutions"]; ok {
Expand Down
11 changes: 0 additions & 11 deletions cli/internal/packagemanager/npm.go
View file
Open in desktop
Expand Up @@ -38,17 +38,6 @@ var nodejsNpm = PackageManager{
}, nil
},

Matches: func(manager string, version string) (bool, error) {
return manager == "npm", nil
},

detect: func(projectDirectory turbopath.AbsoluteSystemPath, packageManager *PackageManager) (bool, error) {
specfileExists := projectDirectory.UntypedJoin(packageManager.Specfile).FileExists()
lockfileExists := projectDirectory.UntypedJoin(packageManager.Lockfile).FileExists()

return (specfileExists && lockfileExists), nil
},

canPrune: func(cwd turbopath.AbsoluteSystemPath) (bool, error) {
return true, nil
},
Expand Down
80 changes: 15 additions & 65 deletions cli/internal/packagemanager/packagemanager.go
View file
Open in desktop
Expand Up @@ -7,15 +7,12 @@ package packagemanager
import (
"fmt"
"path/filepath"
"regexp"
"strings"

"github.com/pkg/errors"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/globby"
"github.com/vercel/turbo/cli/internal/lockfile"
"github.com/vercel/turbo/cli/internal/turbopath"
"github.com/vercel/turbo/cli/internal/util"
)

// PackageManager is an abstraction across package managers
Expand Down Expand Up @@ -54,12 +51,6 @@ type PackageManager struct {
// Detect if Turbo knows how to produce a pruned workspace for the project
canPrune func(cwd turbopath.AbsoluteSystemPath) (bool, error)

// Test a manager and version tuple to see if it is the Package Manager.
Matches func(manager string, version string) (bool, error)

// Detect if the project is using the Package Manager by inspecting the system.
detect func(projectDirectory turbopath.AbsoluteSystemPath, packageManager *PackageManager) (bool, error)

// Read a lockfile for a given package manager
UnmarshalLockfile func(rootPackageJSON *fs.PackageJSON, contents []byte) (lockfile.Lockfile, error)

Expand All @@ -75,63 +66,22 @@ var packageManagers = []PackageManager{
nodejsPnpm6,
}

var (
packageManagerPattern = `(npm|pnpm|yarn)@(\d+)\.\d+\.\d+(-.+)?`
packageManagerRegex = regexp.MustCompile(packageManagerPattern)
)

// ParsePackageManagerString takes a package manager version string parses it into consituent components
func ParsePackageManagerString(packageManager string) (manager string, version string, err error) {
match := packageManagerRegex.FindString(packageManager)
if len(match) == 0 {
return "", "", fmt.Errorf("We could not parse packageManager field in package.json, expected: %s, received: %s", packageManagerPattern, packageManager)
// GetPackageManager reads the package manager name sent by the Rust side
func GetPackageManager(name string) (packageManager *PackageManager, err error) {
switch name {
case "yarn":
return &nodejsYarn, nil
case "berry":
return &nodejsBerry, nil
case "npm":
return &nodejsNpm, nil
case "pnpm":
return &nodejsPnpm, nil
case "pnpm6":
return &nodejsPnpm6, nil
default:
return nil, errors.New("Unknown package manager")
}

return strings.Split(match, "@")[0], strings.Split(match, "@")[1], nil
}

// GetPackageManager attempts all methods for identifying the package manager in use.
func GetPackageManager(projectDirectory turbopath.AbsoluteSystemPath, pkg *fs.PackageJSON) (packageManager *PackageManager, err error) {
result, _ := readPackageManager(pkg)
if result != nil {
return result, nil
}

return detectPackageManager(projectDirectory)
}

// readPackageManager attempts to read the package manager from the package.json.
func readPackageManager(pkg *fs.PackageJSON) (packageManager *PackageManager, err error) {
if pkg.PackageManager != "" {
manager, version, err := ParsePackageManagerString(pkg.PackageManager)
if err != nil {
return nil, err
}

for _, packageManager := range packageManagers {
isResponsible, err := packageManager.Matches(manager, version)
NicholasLYang marked this conversation as resolved.
Show resolved Hide resolved
if isResponsible && (err == nil) {
return &packageManager, nil
}
}
}

return nil, errors.New(util.Sprintf("We did not find a package manager specified in your root package.json. Please set the \"packageManager\" property in your root package.json (${UNDERLINE}https://nodejs.org/api/packages.html#packagemanager)${RESET} or run `npx @turbo/codemod add-package-manager` in the root of your monorepo."))
}

// detectPackageManager attempts to detect the package manager by inspecting the project directory state.
func detectPackageManager(projectDirectory turbopath.AbsoluteSystemPath) (packageManager *PackageManager, err error) {
for _, packageManager := range packageManagers {
isResponsible, err := packageManager.detect(projectDirectory, &packageManager)
NicholasLYang marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
return nil, err
}
if isResponsible {
return &packageManager, nil
}
}

return nil, errors.New(util.Sprintf("We did not detect an in-use package manager for your project. Please set the \"packageManager\" property in your root package.json (${UNDERLINE}https://nodejs.org/api/packages.html#packagemanager)${RESET} or run `npx @turbo/codemod add-package-manager` in the root of your monorepo."))
}

// GetWorkspaces returns the list of package.json files for the current repository.
Expand Down