diff --git a/docs/content/using-npm/config.md b/docs/content/using-npm/config.md index 858fe4d638bed..7bcd7d170f0fd 100644 --- a/docs/content/using-npm/config.md +++ b/docs/content/using-npm/config.md @@ -1161,6 +1161,22 @@ variable will be set to `'production'` for all lifecycle scripts. +#### `omit-lockfile-registry-resolved` + +* Default: false +* Type: Boolean + +Set to true to omit 'resolved' key from registry dependencies in lock files. + +This setting is useful in projects that may install dependencies from +different registries but would use a lockfile to lock package versions. This +option makes installing slower because npm must fetch package manifest to +resolve the package version's tarball. See 'record-default-registry' for an +alternative. + + + + #### `otp` * Default: null @@ -1333,6 +1349,22 @@ Rebuild bundled dependencies after installation. +#### `record-default-registry` + +* Default: false +* Type: Boolean + +Set to true to replace the actual registry in urls resolved from registires +with the default registry when recording lock files. + +This setting is useful in projects that may install dependencies from +different registries but would use a lockfile to lock package versions. This +option supports registries that host tarballs at the same path. If even the +path may change see 'omit-lockfile-registry-resolved'. + + + + #### `registry` * Default: "https://registry.npmjs.org/" diff --git a/lib/utils/config/definitions.js b/lib/utils/config/definitions.js index 316e737091eb4..92f7615d4f4b9 100644 --- a/lib/utils/config/definitions.js +++ b/lib/utils/config/definitions.js @@ -2304,3 +2304,33 @@ define('yes', { the command line. `, }) + +define('omit-lockfile-registry-resolved', { + default: false, + type: Boolean, + description: ` + Set to true to omit 'resolved' key from registry dependencies in lock files. + + This setting is useful in projects that may install dependencies from + different registries but would use a lockfile to lock package versions. + This option makes installing slower because npm must fetch package manifest + to resolve the package version's tarball. + See 'record-default-registry' for an alternative. + `, + flatten, +}) + +define('record-default-registry', { + default: false, + type: Boolean, + description: ` + Set to true to replace the actual registry in urls resolved from registires + with the default registry when recording lock files. + + This setting is useful in projects that may install dependencies from + different registries but would use a lockfile to lock package versions. + This option supports registries that host tarballs at the same path. If + even the path may change see 'omit-lockfile-registry-resolved'. + `, + flatten, +}) diff --git a/tap-snapshots/test/lib/commands/config.js.test.cjs b/tap-snapshots/test/lib/commands/config.js.test.cjs index f98e74c066269..0ef339dd03c46 100644 --- a/tap-snapshots/test/lib/commands/config.js.test.cjs +++ b/tap-snapshots/test/lib/commands/config.js.test.cjs @@ -158,6 +158,8 @@ exports[`test/lib/commands/config.js TAP config list --json > output matches sna "workspaces": null, "workspaces-update": true, "yes": null, + "omit-lockfile-registry-resolved": false, + "record-default-registry": false, "metrics-registry": "https://registry.npmjs.org/" } ` @@ -255,6 +257,7 @@ noproxy = [""] npm-version = "{NPM-VERSION}" offline = false omit = [] +omit-lockfile-registry-resolved = false only = null optional = null otp = null @@ -272,6 +275,7 @@ progress = true proxy = null read-only = false rebuild-bundle = true +record-default-registry = false registry = "https://registry.npmjs.org/" save = true save-bundle = false diff --git a/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs b/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs index b18b8e7a829e6..b02e3f5726753 100644 --- a/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs +++ b/tap-snapshots/test/lib/utils/config/definitions.js.test.cjs @@ -154,6 +154,8 @@ Array [ "workspaces", "workspaces-update", "yes", + "omit-lockfile-registry-resolved", + "record-default-registry", ] ` @@ -1223,6 +1225,21 @@ If the resulting omit list includes \`'dev'\`, then the \`NODE_ENV\` environment variable will be set to \`'production'\` for all lifecycle scripts. ` +exports[`test/lib/utils/config/definitions.js TAP > config description for omit-lockfile-registry-resolved 1`] = ` +#### \`omit-lockfile-registry-resolved\` + +* Default: false +* Type: Boolean + +Set to true to omit 'resolved' key from registry dependencies in lock files. + +This setting is useful in projects that may install dependencies from +different registries but would use a lockfile to lock package versions. This +option makes installing slower because npm must fetch package manifest to +resolve the package version's tarball. See 'record-default-registry' for an +alternative. +` + exports[`test/lib/utils/config/definitions.js TAP > config description for only 1`] = ` #### \`only\` @@ -1414,6 +1431,21 @@ exports[`test/lib/utils/config/definitions.js TAP > config description for rebui Rebuild bundled dependencies after installation. ` +exports[`test/lib/utils/config/definitions.js TAP > config description for record-default-registry 1`] = ` +#### \`record-default-registry\` + +* Default: false +* Type: Boolean + +Set to true to replace the actual registry in urls resolved from registires +with the default registry when recording lock files. + +This setting is useful in projects that may install dependencies from +different registries but would use a lockfile to lock package versions. This +option supports registries that host tarballs at the same path. If even the +path may change see 'omit-lockfile-registry-resolved'. +` + exports[`test/lib/utils/config/definitions.js TAP > config description for registry 1`] = ` #### \`registry\` diff --git a/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs b/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs index 564ade46e731d..072d86a82eaf1 100644 --- a/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs +++ b/tap-snapshots/test/lib/utils/config/describe-all.js.test.cjs @@ -1035,6 +1035,22 @@ variable will be set to \`'production'\` for all lifecycle scripts. +#### \`omit-lockfile-registry-resolved\` + +* Default: false +* Type: Boolean + +Set to true to omit 'resolved' key from registry dependencies in lock files. + +This setting is useful in projects that may install dependencies from +different registries but would use a lockfile to lock package versions. This +option makes installing slower because npm must fetch package manifest to +resolve the package version's tarball. See 'record-default-registry' for an +alternative. + + + + #### \`otp\` * Default: null @@ -1207,6 +1223,22 @@ Rebuild bundled dependencies after installation. +#### \`record-default-registry\` + +* Default: false +* Type: Boolean + +Set to true to replace the actual registry in urls resolved from registires +with the default registry when recording lock files. + +This setting is useful in projects that may install dependencies from +different registries but would use a lockfile to lock package versions. This +option supports registries that host tarballs at the same path. If even the +path may change see 'omit-lockfile-registry-resolved'. + + + + #### \`registry\` * Default: "https://registry.npmjs.org/" diff --git a/workspaces/arborist/lib/arborist/build-ideal-tree.js b/workspaces/arborist/lib/arborist/build-ideal-tree.js index f3166c37e1475..60d1acc0ac4d6 100644 --- a/workspaces/arborist/lib/arborist/build-ideal-tree.js +++ b/workspaces/arborist/lib/arborist/build-ideal-tree.js @@ -327,6 +327,7 @@ Try using the package name instead, e.g: ? Shrinkwrap.reset({ path: this.path, lockfileVersion: this.options.lockfileVersion, + resolveOptions: this.options, }).then(meta => Object.assign(root, { meta })) : this.loadVirtual({ root })) @@ -386,6 +387,7 @@ Try using the package name instead, e.g: const meta = new Shrinkwrap({ path: this.path, lockfileVersion: this.options.lockfileVersion, + resolveOptions: this.options, }) meta.reset() root.meta = meta diff --git a/workspaces/arborist/lib/arborist/load-actual.js b/workspaces/arborist/lib/arborist/load-actual.js index b04fc88f65ccb..67d1c20398301 100644 --- a/workspaces/arborist/lib/arborist/load-actual.js +++ b/workspaces/arborist/lib/arborist/load-actual.js @@ -147,6 +147,7 @@ module.exports = cls => class ActualLoader extends cls { const meta = await Shrinkwrap.load({ path: this[_actualTree].path, hiddenLockfile: true, + resolveOptions: this.options, }) if (meta.loadedFromDisk) { this[_actualTree].meta = meta @@ -155,6 +156,7 @@ module.exports = cls => class ActualLoader extends cls { const meta = await Shrinkwrap.load({ path: this[_actualTree].path, lockfileVersion: this.options.lockfileVersion, + resolveOptions: this.options, }) this[_actualTree].meta = meta return this[_loadActualActually]({ root, ignoreMissing }) diff --git a/workspaces/arborist/lib/arborist/load-virtual.js b/workspaces/arborist/lib/arborist/load-virtual.js index 8a41e7686e7e1..4f2a40126108f 100644 --- a/workspaces/arborist/lib/arborist/load-virtual.js +++ b/workspaces/arborist/lib/arborist/load-virtual.js @@ -57,6 +57,7 @@ module.exports = cls => class VirtualLoader extends cls { const s = await Shrinkwrap.load({ path: this.path, lockfileVersion: this.options.lockfileVersion, + resolveOptions: this.options, }) if (!s.loadedFromDisk && !options.root) { const er = new Error('loadVirtual requires existing shrinkwrap file') diff --git a/workspaces/arborist/lib/node.js b/workspaces/arborist/lib/node.js index 45c288bcf6cf7..f8b4964ff3778 100644 --- a/workspaces/arborist/lib/node.js +++ b/workspaces/arborist/lib/node.js @@ -522,6 +522,18 @@ class Node { return this === this.root || this === this.root.target } + get isRegistryDependency () { + if (this.edgesIn.size === 0) { + return false + } + for (const edge of this.edgesIn) { + if (!npa(edge.spec).registry) { + return false + } + } + return true + } + * ancestry () { for (let anc = this; anc; anc = anc.resolveParent) { yield anc diff --git a/workspaces/arborist/lib/override-resolves.js b/workspaces/arborist/lib/override-resolves.js new file mode 100644 index 0000000000000..f991e6d962389 --- /dev/null +++ b/workspaces/arborist/lib/override-resolves.js @@ -0,0 +1,52 @@ +const npa = require('npm-package-arg') + +function overrideResolves (node, resolved, opts = {}) { + const { + omitLockfileRegistryResolved = false, + recordDefaultRegistry = false, + } = opts + + const isRegistryDependency = node.isRegistryDependency + + // omit the resolved url of registry dependencies. this makes installs slower + // because npm must resolve the url for each package version but it allows + // users to use a lockfile across registries that host tarballs at different + // paths. + if (isRegistryDependency && omitLockfileRegistryResolved) { + return undefined + } + + // replace the configured registry with the default registry. the default + // registry is a magic value meaning the current registry, so recording + // resolved with the default registry allows users to switch to a + // different registry without removing their lockfile. The path portion + // of the resolved url is preserved so this trick only works when the + // different registries host tarballs at the same relative paths. + if (isRegistryDependency && recordDefaultRegistry) { + const scope = npa(node.packageName).scope + const registry = scope && opts[`${scope}:registry`] + ? opts[`${scope}:registry`] + : opts.registry + + // normalize registry url - strip trailing slash. + // TODO improve normalization for both the configured registry and resolved + // url. consider port, protocol, more path normalization. + const normalized = registry.endsWith('/') + ? registry.slice(0, -1) + : registry + + // only replace the host if the resolved url is for the configured + // registry. registries may host tarballs on another server. return + // undefined so npm will re-resolve the url from the current registry when + // it reads the lockfile. + if (resolved.startsWith(normalized)) { + return 'https://registry.npmjs.org' + resolved.slice(normalized.length) + } else { + return undefined + } + } + + return resolved +} + +module.exports = { overrideResolves } diff --git a/workspaces/arborist/lib/shrinkwrap.js b/workspaces/arborist/lib/shrinkwrap.js index ab6c91935c78e..cf20afa216d6b 100644 --- a/workspaces/arborist/lib/shrinkwrap.js +++ b/workspaces/arborist/lib/shrinkwrap.js @@ -95,6 +95,7 @@ const specFromResolved = resolved => { const relpath = require('./relpath.js') const consistentResolve = require('./consistent-resolve.js') +const { overrideResolves } = require('./override-resolves.js') const maybeReadFile = file => { return readFile(file, 'utf8').then(d => d, er => { @@ -265,7 +266,7 @@ class Shrinkwrap { return s } - static metaFromNode (node, path) { + static metaFromNode (node, path, options = {}) { if (node.isLink) { return { resolved: relpath(path, node.realpath), @@ -300,7 +301,7 @@ class Shrinkwrap { const resolved = consistentResolve(node.resolved, node.path, path, true) if (resolved) { - meta.resolved = resolved + meta.resolved = overrideResolves(node, resolved, options) } if (node.extraneous) { @@ -330,6 +331,7 @@ class Shrinkwrap { shrinkwrapOnly = false, hiddenLockfile = false, lockfileVersion, + resolveOptions = {}, } = options this.lockfileVersion = hiddenLockfile ? 3 @@ -347,6 +349,7 @@ class Shrinkwrap { this.yarnLock = null this.hiddenLockfile = hiddenLockfile this.loadingError = null + this.resolveOptions = resolveOptions // only load npm-shrinkwrap.json in dep trees, not package-lock this.shrinkwrapOnly = shrinkwrapOnly } @@ -830,7 +833,7 @@ class Shrinkwrap { resolved, integrity, hasShrinkwrap, - } = Shrinkwrap.metaFromNode(node, this.path) + } = Shrinkwrap.metaFromNode(node, this.path, this.resolveOptions) node.resolved = node.resolved || resolved || null node.integrity = node.integrity || integrity || null node.hasShrinkwrap = node.hasShrinkwrap || hasShrinkwrap || false @@ -886,7 +889,10 @@ class Shrinkwrap { [_updateWaitingNode] (loc) { const node = this[_awaitingUpdate].get(loc) this[_awaitingUpdate].delete(loc) - this.data.packages[loc] = Shrinkwrap.metaFromNode(node, this.path) + this.data.packages[loc] = Shrinkwrap.metaFromNode( + node, + this.path, + this.resolveOptions) } commit () { @@ -894,7 +900,10 @@ class Shrinkwrap { if (this.yarnLock) { this.yarnLock.fromTree(this.tree) } - const root = Shrinkwrap.metaFromNode(this.tree.target, this.path) + const root = Shrinkwrap.metaFromNode( + this.tree.target, + this.path, + this.resolveOptions) this.data.packages = {} if (Object.keys(root).length) { this.data.packages[''] = root @@ -905,7 +914,10 @@ class Shrinkwrap { continue } const loc = relpath(this.path, node.path) - this.data.packages[loc] = Shrinkwrap.metaFromNode(node, this.path) + this.data.packages[loc] = Shrinkwrap.metaFromNode( + node, + this.path, + this.resolveOptions) } } else if (this[_awaitingUpdate].size > 0) { for (const loc of this[_awaitingUpdate].keys()) { @@ -1013,7 +1025,7 @@ class Shrinkwrap { spec.type !== 'git' && spec.type !== 'file' && spec.type !== 'remote') { - lock.resolved = node.resolved + lock.resolved = overrideResolves(node, node.resolved, this.resolveOptions) } if (node.integrity) { diff --git a/workspaces/arborist/test/node.js b/workspaces/arborist/test/node.js index ecdf72c22a6dc..29f23aafba3c2 100644 --- a/workspaces/arborist/test/node.js +++ b/workspaces/arborist/test/node.js @@ -2842,3 +2842,26 @@ t.test('overrides', (t) => { t.end() }) + +t.test('node with no edges in is not a registry dep', async t => { + const node = new Node({ path: '/foo' }) + t.equal(node.isRegistryDependency, false) +}) + +t.test('node with non registry edge in is not a registry dep', async t => { + const root = new Node({ path: '/some/path', pkg: { dependencies: { registry: '', tar: '' } } }) + const node = new Node({ pkg: { name: 'node', version: '1.0.0' }, parent: root }) + + new Node({ pkg: { name: 'registry', dependencies: { node: '^1.0.0' } }, parent: root }) + new Node({ pkg: { name: 'tar', dependencies: { node: 'file:node' } }, parent: root }) + + t.equal(node.isRegistryDependency, false) +}) + +t.test('node with only registry edges in a registry dep', async t => { + const root = new Node({ path: '/some/path', pkg: { dependencies: { registry: '', tar: '' } } }) + const node = new Node({ pkg: { name: 'node', version: '1.0.0' }, parent: root }) + new Node({ pkg: { name: 'registry', dependencies: { node: '^1.0.0' } }, parent: root }) + + t.equal(node.isRegistryDependency, true) +}) diff --git a/workspaces/arborist/test/shrinkwrap.js b/workspaces/arborist/test/shrinkwrap.js index d47266d30e1c1..de3055822d988 100644 --- a/workspaces/arborist/test/shrinkwrap.js +++ b/workspaces/arborist/test/shrinkwrap.js @@ -231,6 +231,121 @@ t.test('throws when attempting to access data before loading', t => { t.end() }) +t.only('resolveOptions', async t => { + const url = 'https://private.registry.org/deadbeef/registry/-/registry-1.2.3.tgz' + const someOtherRegistry = 'https://someother.registry.org/registry/-/registry-1.2.3.tgz' + const getData = async (resolveOptions) => { + const dir = t.testdir() + const meta = await Shrinkwrap.load({ + path: dir, + resolveOptions, + }) + + const root = new Node({ + pkg: { + name: 'root', + dependencies: { + registry: '^1.0.0', + 'some-other-registry': '^1.0.0', + '@scope/some-package': '^1.0.0', + tar: url, + }, + }, + path: dir, + realpath: dir, + meta, + }) + + const registry = new Node({ + pkg: { name: 'registry', version: '1.2.3' }, + resolved: url, + integrity: 'sha512-registry', + parent: root, + }) + + const otherRegistry = new Node({ + pkg: { name: 'some-other-registry', version: '1.2.3' }, + resolved: someOtherRegistry, + integrity: 'sha512-registry', + parent: root, + }) + + const scopedOtherRegistry = new Node({ + pkg: { name: '@scope/some-package', version: '1.2.3' }, + resolved: someOtherRegistry, + integrity: 'sha512-registry', + parent: root, + }) + + const tar = new Node({ + pkg: { name: 'tar', version: '1.2.3' }, + resolved: url, + integrity: 'sha512-registry', + parent: root, + }) + + calcDepFlags(root) + meta.add(root) + return { data: meta.commit(), registry, tar, root, otherRegistry, scopedOtherRegistry } + } + + await t.test('omitLockfileRegistryResolved', async t => { + const { data } = await getData({ omitLockfileRegistryResolved: true }) + // registry dependencies in v2 packages and v1 dependencies should + // have resolved stripped. + t.strictSame(data.packages['node_modules/registry'].resolved, undefined) + t.strictSame(data.dependencies.registry.resolved, undefined) + + // tar should have resolved because it is not a registry dep. + t.strictSame(data.packages['node_modules/tar'].resolved, url) + // v1 url dependencies never have resolved. + t.strictSame(data.dependencies.tar.resolved, undefined) + }) + + await t.test('omitLockfileRegistryResolved: false', async t => { + const { data } = await getData({ omitLockfileRegistryResolved: false }) + t.strictSame(data.packages['node_modules/registry'].resolved, url) + t.strictSame(data.dependencies.registry.resolved, url) + + t.strictSame(data.packages['node_modules/tar'].resolved, url) + // v1 url dependencies never have resolved. + t.strictSame(data.dependencies.tar.resolved, undefined) + }) + + await t.test('recordDefaultRegistry: true', async t => { + const { data } = await getData({ + recordDefaultRegistry: true, + registry: 'https://private.registry.org/deadbeef', + '@scope:registry': 'https://someother.registry.org', + }) + + // unscoped packages that resolve to their configured registry should be + // record the default registry + t.strictSame(data.packages['node_modules/registry'].resolved, + 'https://registry.npmjs.org/registry/-/registry-1.2.3.tgz') + t.strictSame(data.dependencies.registry.resolved, + 'https://registry.npmjs.org/registry/-/registry-1.2.3.tgz') + + // scoped packages that resolve to their configured registry should be + // record the default registry + t.strictSame(data.packages['node_modules/@scope/some-package'].resolved, + 'https://registry.npmjs.org/registry/-/registry-1.2.3.tgz') + t.strictSame(data.dependencies['@scope/some-package'].resolved, + 'https://registry.npmjs.org/registry/-/registry-1.2.3.tgz') + + // packages with resolved urls that don't match the configured registry + // should record undefined so npm resolves their url again. + t.strictSame(data.packages['node_modules/some-other-registry'].resolved, undefined) + t.strictSame(data.dependencies['some-other-registry'].resolved, undefined) + }) + + t.test('metaFromNode default', async t => { + // test to cover options default. + const { registry } = await getData(undefined) + t.strictSame(Shrinkwrap.metaFromNode(registry, '').resolved, url) + }) +}) + t.test('construct metadata from node and package data', t => { const meta = new Shrinkwrap({ path: '/home/user/projects/root' }) // fake load