From 6f0550530d8ca823289312ddee512178272c7758 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Mon, 17 Apr 2023 08:43:31 -0700 Subject: [PATCH 1/2] delete unused npm lockfile impl in go --- cli/internal/lockfile/lockfile.go | 13 + .../{npm_lockfile_rust.go => npm_lockfile.go} | 3 - cli/internal/lockfile/npm_lockfile_go.go | 276 ------------------ .../lockfile/transitive_closure_go.go | 18 -- .../lockfile/transitive_closure_rust.go | 22 -- 5 files changed, 13 insertions(+), 319 deletions(-) rename cli/internal/lockfile/{npm_lockfile_rust.go => npm_lockfile.go} (99%) delete mode 100644 cli/internal/lockfile/npm_lockfile_go.go delete mode 100644 cli/internal/lockfile/transitive_closure_go.go delete mode 100644 cli/internal/lockfile/transitive_closure_rust.go diff --git a/cli/internal/lockfile/lockfile.go b/cli/internal/lockfile/lockfile.go index 9ca8182ea83b6..bb36eda93f419 100644 --- a/cli/internal/lockfile/lockfile.go +++ b/cli/internal/lockfile/lockfile.go @@ -61,6 +61,19 @@ func (p ByKey) Less(i, j int) bool { var _ (sort.Interface) = (*ByKey)(nil) +// TransitiveClosure the set of all lockfile keys that pkg depends on +func TransitiveClosure( + workspaceDir turbopath.AnchoredUnixPath, + unresolvedDeps map[string]string, + lockFile Lockfile, +) (mapset.Set, error) { + if lf, ok := lockFile.(*NpmLockfile); ok { + // We special case as Rust implementations have their own dep crawl + return npmTransitiveDeps(lf, workspaceDir, unresolvedDeps) + } + return transitiveClosure(workspaceDir, unresolvedDeps, lockFile) +} + func transitiveClosure( workspaceDir turbopath.AnchoredUnixPath, unresolvedDeps map[string]string, diff --git a/cli/internal/lockfile/npm_lockfile_rust.go b/cli/internal/lockfile/npm_lockfile.go similarity index 99% rename from cli/internal/lockfile/npm_lockfile_rust.go rename to cli/internal/lockfile/npm_lockfile.go index 6704fc7180aed..67cd32af63897 100644 --- a/cli/internal/lockfile/npm_lockfile_rust.go +++ b/cli/internal/lockfile/npm_lockfile.go @@ -1,6 +1,3 @@ -//go:build rust -// +build rust - package lockfile import ( diff --git a/cli/internal/lockfile/npm_lockfile_go.go b/cli/internal/lockfile/npm_lockfile_go.go deleted file mode 100644 index 57ba7aaed6f24..0000000000000 --- a/cli/internal/lockfile/npm_lockfile_go.go +++ /dev/null @@ -1,276 +0,0 @@ -//go:build go || !rust -// +build go !rust - -package lockfile - -import ( - "encoding/json" - "fmt" - "io" - "strings" - - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// NpmLockfile representation of package-lock.json -type NpmLockfile struct { - Name string `json:"name"` - Version string `json:"version"` - LockfileVersion int `json:"lockfileVersion,omitempty"` - Requires bool `json:"requires,omitempty"` - // Keys are paths to package.json, can be nested in node_modules - Packages map[string]NpmPackage `json:"packages,omitempty"` - // Legacy info for npm version 5 & 6 - Dependencies map[string]NpmDependency `json:"dependencies,omitempty"` -} - -// NpmPackage Representation of dependencies used in LockfileVersion 2+ -type NpmPackage struct { - // Only used for root level package - Name string `json:"name,omitempty"` - - Version string `json:"version,omitempty"` - Resolved string `json:"resolved,omitempty"` - Integrity string `json:"integrity,omitempty"` - Link bool `json:"link,omitempty"` - - Dev bool `json:"dev,omitempty"` - Optional bool `json:"optional,omitempty"` - // Only used if included as a dev dep of one package and an optional dep of another - // An optional dep of a dev dep will have dev and optional set to true - DevOptional bool `json:"devOptional,omitempty"` - InBundle bool `json:"inBundle,omitempty"` - HasInstallScript bool `json:"hasInstallScript,omitempty"` - HasShrinkwrap bool `json:"hasShrinkwrap,omitempty"` - Extraneous bool `json:"extraneous,omitempty"` - - Dependencies map[string]string `json:"dependencies,omitempty"` - DevDependencies map[string]string `json:"devDependencies,omitempty"` - PeerDependencies map[string]string `json:"peerDependencies,omitempty"` - PeerDependenciesMeta map[string]struct { - Optional bool `json:"optional,omitempty"` - } `json:"peerDependenciesMeta,omitempty"` - OptionalDependencies map[string]string `json:"optionalDependencies,omitempty"` - - Bin map[string]string `json:"bin,omitempty"` - - // Engines has two valid formats historically, an array and an object. - // Since we don't use this property we just need to propagate it through. - // - // Go serializes correctly even with `interface{}`. - Engines interface{} `json:"engines,omitempty"` - - CPU []string `json:"cpu,omitempty"` - OS []string `json:"os,omitempty"` - - // Only used for root level package - Workspaces Workspaces `json:"workspaces,omitempty"` -} - -// NpmDependency Legacy representation of dependencies -type NpmDependency struct { - Version string `json:"version"` - Resolved string `json:"resolved,omitempty"` - Integrity string `json:"integrity,omitempty"` - Bundled bool `json:"bundled,omitempty"` - Dev bool `json:"dev,omitempty"` - Optional bool `json:"optional,omitempty"` - Requires map[string]string `json:"requires,omitempty"` - Dependencies map[string]NpmDependency `json:"dependencies,omitempty"` -} - -// Workspaces represents the standard workspaces field in package.json -type Workspaces []string - -// WorkspacesAlt represents the alternate workspaces field (nested) package.json -type WorkspacesAlt struct { - Packages []string `json:"packages,omitempty"` -} - -// UnmarshalJSON determines the correct format of the workspaces field to unmarshal -func (r *Workspaces) UnmarshalJSON(data []byte) error { - var tmp = &WorkspacesAlt{} - if err := json.Unmarshal(data, tmp); err == nil { - *r = Workspaces(tmp.Packages) - return nil - } - var tempstr = []string{} - if err := json.Unmarshal(data, &tempstr); err != nil { - return err - } - *r = tempstr - return nil -} - -var _ Lockfile = (*NpmLockfile)(nil) - -// ResolvePackage Given a workspace, a package it imports and version returns the key, resolved version, and if it was found -func (l *NpmLockfile) ResolvePackage(workspacePath turbopath.AnchoredUnixPath, name string, version string) (Package, error) { - _, ok := l.Packages[workspacePath.ToString()] - if !ok { - return Package{}, fmt.Errorf("No package found in lockfile for '%s'", workspacePath) - } - - // AllDependencies will return a key to avoid choosing the incorrect transitive dep - if entry, ok := l.Packages[name]; ok { - return Package{ - Key: name, - Version: entry.Version, - Found: true, - }, nil - } - - // If we didn't find the entry just using name, then this is an initial call to ResolvePackage - // based on information coming from internal packages' package.json - // First we check if the workspace uses a nested version of the package - nestedPath := fmt.Sprintf("%s/node_modules/%s", workspacePath, name) - if entry, ok := l.Packages[nestedPath]; ok { - return Package{ - Key: nestedPath, - Version: entry.Version, - Found: true, - }, nil - } - - // Next we check for a top level version of the package - hoistedPath := fmt.Sprintf("node_modules/%s", name) - if entry, ok := l.Packages[hoistedPath]; ok { - return Package{ - Key: hoistedPath, - Version: entry.Version, - Found: true, - }, nil - } - - return Package{}, nil -} - -// AllDependencies Given a lockfile key return all (dev/optional/peer) dependencies of that package -func (l *NpmLockfile) AllDependencies(key string) (map[string]string, bool) { - entry, ok := l.Packages[key] - if !ok { - return nil, false - } - deps := make(map[string]string, len(entry.Dependencies)+len(entry.DevDependencies)+len(entry.PeerDependencies)+len(entry.OptionalDependencies)) - addDep := func(d map[string]string) { - for name := range d { - for _, possibleKey := range possibleNpmDeps(key, name) { - if entry, ok := l.Packages[possibleKey]; ok { - deps[possibleKey] = entry.Version - break - } - } - } - } - - addDep(entry.Dependencies) - addDep(entry.DevDependencies) - addDep(entry.OptionalDependencies) - addDep(entry.PeerDependencies) - - return deps, true -} - -// Subgraph Given a list of lockfile keys returns a Lockfile based off the original one that only contains the packages given -func (l *NpmLockfile) Subgraph(workspacePackages []turbopath.AnchoredSystemPath, packages []string) (Lockfile, error) { - prunedPackages := make(map[string]NpmPackage, len(packages)) - for _, pkg := range packages { - if entry, ok := l.Packages[pkg]; ok { - prunedPackages[pkg] = entry - } else { - return nil, fmt.Errorf("No lockfile entry found for %s", pkg) - } - } - if rootEntry, ok := l.Packages[""]; ok { - prunedPackages[""] = rootEntry - } - for _, workspacePackages := range workspacePackages { - workspacePkg := workspacePackages.ToUnixPath().ToString() - if workspaceEntry, ok := l.Packages[workspacePkg]; ok { - prunedPackages[workspacePkg] = workspaceEntry - } else { - return nil, fmt.Errorf("No lockfile entry found for %s", workspacePkg) - } - // Each workspace package has a fake version of the package that links back to the original - // but is needed for dependency resolution. - for key, entry := range l.Packages { - if entry.Resolved == workspacePkg { - prunedPackages[key] = entry - break - } - } - } - - return &NpmLockfile{ - Name: l.Name, - Version: l.Version, - // Forcible upgrade to version 3 since we aren't including backward compatibility capabilities - LockfileVersion: 3, - Requires: l.Requires, - Packages: prunedPackages, - // We omit dependencies since they are only used for supporting npm <=6 - }, nil -} - -// Encode the lockfile representation and write it to the given writer -func (l *NpmLockfile) Encode(w io.Writer) error { - encoder := json.NewEncoder(w) - encoder.SetIndent("", " ") - encoder.SetEscapeHTML(false) - return encoder.Encode(l) -} - -// Patches return a list of patches used in the lockfile -func (l *NpmLockfile) Patches() []turbopath.AnchoredUnixPath { - return nil -} - -// DecodeNpmLockfile Parse contents of package-lock.json into NpmLockfile -func DecodeNpmLockfile(content []byte) (*NpmLockfile, error) { - var lockfile NpmLockfile - if err := json.Unmarshal(content, &lockfile); err != nil { - return nil, err - } - - // LockfileVersion <=1 is for npm <=6 - ancientLockfile := lockfile.LockfileVersion <= 1 || (len(lockfile.Dependencies) > 0 && len(lockfile.Packages) == 0) - - if ancientLockfile { - // TODO: Older versions of package-lock.json required crawling through node_modules and other - // various fixups to make it a deterministic lockfile. - // See https://github.com/npm/cli/blob/9609e9eed87c735f0319ac0af265f4d406cbf800/workspaces/arborist/lib/shrinkwrap.js#L674 - return nil, fmt.Errorf("Support for lockfiles without a 'packages' field isn't implemented yet") - } - - return &lockfile, nil -} - -// GlobalChange checks if there are any differences between lockfiles that would completely invalidate -// the cache. -func (l *NpmLockfile) GlobalChange(other Lockfile) bool { - o, ok := other.(*NpmLockfile) - return !ok || - l.LockfileVersion != o.LockfileVersion || - l.Requires != o.Requires -} - -// returns a list of possible keys for a dependency of package key -func possibleNpmDeps(key string, dep string) []string { - possibleDeps := []string{fmt.Sprintf("%s/node_modules/%s", key, dep)} - - curr := key - for curr != "" { - next := npmPathParent(curr) - possibleDeps = append(possibleDeps, fmt.Sprintf("%snode_modules/%s", next, dep)) - curr = next - } - - return possibleDeps -} - -func npmPathParent(key string) string { - if index := strings.LastIndex(key, "node_modules/"); index != -1 { - return key[0:index] - } - return "" -} diff --git a/cli/internal/lockfile/transitive_closure_go.go b/cli/internal/lockfile/transitive_closure_go.go deleted file mode 100644 index e07151cd722db..0000000000000 --- a/cli/internal/lockfile/transitive_closure_go.go +++ /dev/null @@ -1,18 +0,0 @@ -//go:build go || !rust -// +build go !rust - -package lockfile - -import ( - mapset "github.com/deckarep/golang-set" - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// TransitiveClosure the set of all lockfile keys that pkg depends on -func TransitiveClosure( - workspaceDir turbopath.AnchoredUnixPath, - unresolvedDeps map[string]string, - lockFile Lockfile, -) (mapset.Set, error) { - return transitiveClosure(workspaceDir, unresolvedDeps, lockFile) -} diff --git a/cli/internal/lockfile/transitive_closure_rust.go b/cli/internal/lockfile/transitive_closure_rust.go deleted file mode 100644 index 4306f80b478d5..0000000000000 --- a/cli/internal/lockfile/transitive_closure_rust.go +++ /dev/null @@ -1,22 +0,0 @@ -//go:build rust -// +build rust - -package lockfile - -import ( - mapset "github.com/deckarep/golang-set" - "github.com/vercel/turbo/cli/internal/turbopath" -) - -// TransitiveClosure the set of all lockfile keys that pkg depends on -func TransitiveClosure( - workspaceDir turbopath.AnchoredUnixPath, - unresolvedDeps map[string]string, - lockFile Lockfile, -) (mapset.Set, error) { - if lf, ok := lockFile.(*NpmLockfile); ok { - // We special case as Rust implementations have their own dep crawl - return npmTransitiveDeps(lf, workspaceDir, unresolvedDeps) - } - return transitiveClosure(workspaceDir, unresolvedDeps, lockFile) -} From 9c0eea08b3d55e7b94ddc6ca00ce9046dc491cd0 Mon Sep 17 00:00:00 2001 From: Chris Olszewski Date: Mon, 17 Apr 2023 08:44:39 -0700 Subject: [PATCH 2/2] delete go unit tests --- cli/internal/lockfile/npm_lockfile_test.go | 255 --------------------- 1 file changed, 255 deletions(-) delete mode 100644 cli/internal/lockfile/npm_lockfile_test.go diff --git a/cli/internal/lockfile/npm_lockfile_test.go b/cli/internal/lockfile/npm_lockfile_test.go deleted file mode 100644 index c7eecc14f0f1b..0000000000000 --- a/cli/internal/lockfile/npm_lockfile_test.go +++ /dev/null @@ -1,255 +0,0 @@ -//go:build go || !rust -// +build go !rust - -package lockfile - -import ( - "bytes" - "sort" - "strings" - "testing" - - "github.com/vercel/turbo/cli/internal/turbopath" - "gotest.tools/v3/assert" - "gotest.tools/v3/assert/cmp" -) - -func getNpmLockfile(t *testing.T, file string) *NpmLockfile { - content, err := getFixture(t, file) - assert.NilError(t, err, "reading {}", file) - lockfile, err := DecodeNpmLockfile(content) - assert.NilError(t, err, "parsing {}", file) - return lockfile -} - -func Test_NpmPathParent(t *testing.T) { - type TestCase struct { - key string - parent string - } - testCases := []TestCase{ - { - key: "apps/docs", - parent: "", - }, - { - key: "apps/docs/node_modules/foo", - parent: "apps/docs/", - }, - { - key: "node_modules/foo", - parent: "", - }, - { - key: "node_modules/foo/node_modules/bar", - parent: "node_modules/foo/", - }, - } - - for _, tc := range testCases { - assert.Equal(t, npmPathParent(tc.key), tc.parent, tc.key) - } -} - -func Test_NpmResolvesAlternateWorkspaceFormat(t *testing.T) { - lockfile := getNpmLockfile(t, "npm-lock-workspace-variation.json") - assert.Equal(t, lockfile.Name, "npm-prune-workspace-variation") -} - -func Test_PossibleNpmDeps(t *testing.T) { - type TestCase struct { - name string - key string - dep string - expected []string - } - testCases := []TestCase{ - { - name: "top level looks for children", - key: "node_modules/foo", - dep: "baz", - expected: []string{ - "node_modules/foo/node_modules/baz", - "node_modules/baz", - }, - }, - { - name: "if child looks for siblings", - key: "node_modules/foo/node_modules/bar", - dep: "baz", - expected: []string{ - "node_modules/foo/node_modules/bar/node_modules/baz", - "node_modules/foo/node_modules/baz", - "node_modules/baz", - }, - }, - { - name: "deeply nested package looks through all ancestors", - key: "node_modules/foo1/node_modules/foo2/node_modules/foo3/node_modules/foo4", - dep: "bar", - expected: []string{ - "node_modules/foo1/node_modules/foo2/node_modules/foo3/node_modules/foo4/node_modules/bar", - "node_modules/foo1/node_modules/foo2/node_modules/foo3/node_modules/bar", - "node_modules/foo1/node_modules/foo2/node_modules/bar", - "node_modules/foo1/node_modules/bar", - "node_modules/bar", - }, - }, - { - name: "workspace deps look for nested", - key: "apps/docs/node_modules/foo", - dep: "baz", - expected: []string{ - "apps/docs/node_modules/foo/node_modules/baz", - "apps/docs/node_modules/baz", - "node_modules/baz", - }, - }, - } - - for _, tc := range testCases { - actual := possibleNpmDeps(tc.key, tc.dep) - assert.Assert(t, cmp.DeepEqual(actual, tc.expected), tc.name) - } -} - -func Test_NpmResolvePackage(t *testing.T) { - type TestCase struct { - testName string - workspace string - name string - key string - version string - err string - } - testCases := []TestCase{ - { - testName: "finds deps of root package", - workspace: "", - name: "turbo", - key: "node_modules/turbo", - version: "1.5.5", - }, - { - testName: "selects nested dep if present", - workspace: "apps/web", - name: "lodash", - key: "apps/web/node_modules/lodash", - version: "4.17.21", - }, - { - testName: "selects top level package if no nested package", - workspace: "apps/docs", - name: "lodash", - key: "node_modules/lodash", - version: "3.10.1", - }, - { - testName: "finds package if given resolved key", - workspace: "apps/docs", - name: "node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping", - key: "node_modules/@babel/generator/node_modules/@jridgewell/gen-mapping", - version: "0.3.2", - }, - } - - lockfile := getNpmLockfile(t, "npm-lock.json") - for _, tc := range testCases { - workspace := turbopath.AnchoredUnixPath(tc.workspace) - pkg, err := lockfile.ResolvePackage(workspace, tc.name, "") - if tc.err != "" { - assert.Error(t, err, tc.err) - } else { - assert.NilError(t, err, tc.testName) - } - assert.Assert(t, pkg.Found, tc.testName) - assert.Equal(t, pkg.Key, tc.key, tc.testName) - assert.Equal(t, pkg.Version, tc.version, tc.testName) - } -} - -func Test_NpmAllDependencies(t *testing.T) { - type TestCase struct { - name string - key string - expected []string - } - testCases := []TestCase{ - { - name: "mixed nested and hoisted", - key: "node_modules/table", - expected: []string{ - "node_modules/lodash.truncate", - "node_modules/slice-ansi", - "node_modules/string-width", - "node_modules/strip-ansi", - "node_modules/table/node_modules/ajv", - }, - }, - { - name: "deps of nested packaged", - key: "node_modules/table/node_modules/ajv", - expected: []string{ - "node_modules/fast-deep-equal", - "node_modules/require-from-string", - "node_modules/table/node_modules/json-schema-traverse", - "node_modules/uri-js", - }, - }, - } - - lockfile := getNpmLockfile(t, "npm-lock.json") - for _, tc := range testCases { - deps, ok := lockfile.AllDependencies(tc.key) - assert.Assert(t, ok, tc.name) - depKeys := make([]string, len(deps)) - i := 0 - for dep := range deps { - depKeys[i] = dep - i++ - } - sort.Strings(depKeys) - assert.DeepEqual(t, depKeys, tc.expected) - } - -} - -func Test_NpmPeerDependenciesMeta(t *testing.T) { - var buf bytes.Buffer - - lockfile := getNpmLockfile(t, "npm-lock.json") - if err := lockfile.Encode(&buf); err != nil { - t.Error(err) - } - s := buf.String() - - expected := `"node_modules/eslint-config-next": { - "version": "12.3.1", - "resolved": "https://registry.npmjs.org/eslint-config-next/-/eslint-config-next-12.3.1.tgz", - "integrity": "sha512-EN/xwKPU6jz1G0Qi6Bd/BqMnHLyRAL0VsaQaWA7F3KkjAgZHi4f1uL1JKGWNxdQpHTW/sdGONBd0bzxUka/DJg==", - "dependencies": { - "@next/eslint-plugin-next": "12.3.1", - "@rushstack/eslint-patch": "^1.1.3", - "@typescript-eslint/parser": "^5.21.0", - "eslint-import-resolver-node": "^0.3.6", - "eslint-import-resolver-typescript": "^2.7.1", - "eslint-plugin-import": "^2.26.0", - "eslint-plugin-jsx-a11y": "^6.5.1", - "eslint-plugin-react": "^7.31.7", - "eslint-plugin-react-hooks": "^4.5.0" - }, - "peerDependencies": { - "eslint": "^7.23.0 || ^8.0.0", - "typescript": ">=3.3.1" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - },` - - if !strings.Contains(s, expected) { - t.Error("failed to persist \"peerDependenciesMeta\" in npm lockfile") - } -}