diff --git a/package.json b/package.json index 405c7b2790a1d..ee737c7c84d03 100644 --- a/package.json +++ b/package.json @@ -111,6 +111,7 @@ "@types/react-dom": "17.0.3", "@types/react-router-dom": "5.1.7", "@types/semver": "^7.3.8", + "@types/tar-stream": "^2.2.2", "@types/tmp": "^0.2.0", "@types/yargs": "^17.0.10", "@typescript-eslint/eslint-plugin": "5.10.1", @@ -230,6 +231,7 @@ "styled-components": "5.0.0", "stylus": "^0.55.0", "stylus-loader": "^6.2.0", + "tar-stream": "~2.2.0", "tcp-port-used": "^1.0.2", "terser-webpack-plugin": "^5.3.0", "tmp": "~0.2.1", diff --git a/packages/make-angular-cli-faster/src/utilities/migration.ts b/packages/make-angular-cli-faster/src/utilities/migration.ts index 9398d6f632f71..bc704bda6c357 100644 --- a/packages/make-angular-cli-faster/src/utilities/migration.ts +++ b/packages/make-angular-cli-faster/src/utilities/migration.ts @@ -1,7 +1,11 @@ -import { getPackageManagerCommand, output, readJsonFile } from '@nrwl/devkit'; +import { + getPackageManagerCommand, + output, + readJsonFile, + workspaceRoot, +} from '@nrwl/devkit'; import { execSync } from 'child_process'; import { prompt } from 'enquirer'; -import { appRootPath } from 'nx/src/utils/app-root'; import { lt, lte, major, satisfies } from 'semver'; import { resolvePackageVersion } from './package-manager'; import { MigrationDefinition } from './types'; @@ -168,7 +172,7 @@ async function promptForVersion(version: string): Promise { function getInstalledAngularVersion(): string { const packageJsonPath = require.resolve('@angular/core/package.json', { - paths: [appRootPath], + paths: [workspaceRoot], }); return readJsonFile(packageJsonPath).version; } diff --git a/packages/make-angular-cli-faster/src/utilities/package-manager.ts b/packages/make-angular-cli-faster/src/utilities/package-manager.ts index 1f6b6543f1384..f178d154fb29f 100644 --- a/packages/make-angular-cli-faster/src/utilities/package-manager.ts +++ b/packages/make-angular-cli-faster/src/utilities/package-manager.ts @@ -1,15 +1,17 @@ import { getPackageManagerCommand, readJsonFile, - writeJsonFile, + workspaceRoot, } from '@nrwl/devkit'; import { execSync } from 'child_process'; -import { copyFileSync, existsSync, unlinkSync, writeFileSync } from 'fs'; -import { appRootPath } from 'nx/src/utils/app-root'; +import { writeFileSync } from 'fs'; import { sortObjectByKeys } from 'nx/src/utils/object-sort'; -import { dirname, join } from 'path'; +import { + resolvePackageVersionUsingInstallation, + resolvePackageVersionUsingRegistry, +} from 'nx/src/utils/package-manager'; +import { join } from 'path'; import { gte, major } from 'semver'; -import { dirSync } from 'tmp'; import { MigrationDefinition } from './types'; // version when the Nx CLI changed from @nrwl/tao & @nrwl/cli to nx @@ -19,7 +21,7 @@ export function installDependencies( { packageName, version }: MigrationDefinition, useNxCloud: boolean ): void { - const json = readJsonFile(join(appRootPath, 'package.json')); + const json = readJsonFile(join(workspaceRoot, 'package.json')); json.devDependencies ??= {}; json.devDependencies['@nrwl/workspace'] = version; @@ -57,37 +59,9 @@ export function resolvePackageVersion( packageName: string, version: string ): string { - const dir = dirSync().name; - const npmrc = checkForNPMRC(); - if (npmrc) { - // Creating a package.json is needed for .npmrc to resolve - writeJsonFile(`${dir}/package.json`, {}); - // Copy npmrc if it exists, so that npm still follows it. - copyFileSync(npmrc, `${dir}/.npmrc`); - } - - const pmc = getPackageManagerCommand(); - execSync(`${pmc.add} ${packageName}@${version}`, { stdio: [], cwd: dir }); - - const packageJsonPath = require.resolve(`${packageName}/package.json`, { - paths: [dir], - }); - const { version: resolvedVersion } = readJsonFile(packageJsonPath); - try { - unlinkSync(dir); + return resolvePackageVersionUsingRegistry(packageName, version); } catch { - // It's okay if this fails, the OS will clean it up eventually - } - - return resolvedVersion; -} - -function checkForNPMRC(): string | null { - let directory = process.cwd(); - while (!existsSync(join(directory, 'package.json'))) { - directory = dirname(directory); + return resolvePackageVersionUsingInstallation(packageName, version); } - const path = join(directory, '.npmrc'); - return existsSync(path) ? path : null; } diff --git a/packages/nx/package.json b/packages/nx/package.json index b8d8864f8db7e..eb75e33f9cd6a 100644 --- a/packages/nx/package.json +++ b/packages/nx/package.json @@ -51,6 +51,7 @@ "open": "^8.4.0", "rxjs": "^6.5.4", "semver": "7.3.4", + "tar-stream": "~2.2.0", "tmp": "~0.2.1", "yargs": "^17.4.0", "yargs-parser": "21.0.1", diff --git a/packages/nx/src/command-line/migrate.ts b/packages/nx/src/command-line/migrate.ts index f815da3dba123..b9cc727a6c822 100644 --- a/packages/nx/src/command-line/migrate.ts +++ b/packages/nx/src/command-line/migrate.ts @@ -1,18 +1,24 @@ import { execSync } from 'child_process'; -import { copyFileSync, existsSync, removeSync } from 'fs-extra'; +import { copyFileSync, removeSync } from 'fs-extra'; import { dirname, join } from 'path'; import { gt, lte } from 'semver'; import { dirSync } from 'tmp'; -import { logger } from '../utils/logger'; -import { handleErrors } from '../utils/params'; -import { getPackageManagerCommand } from '../utils/package-manager'; +import { NxJsonConfiguration } from '../shared/nx'; import { flushChanges, FsTree } from '../shared/tree'; import { + extractFileFromTarball, JsonReadOptions, readJsonFile, writeJsonFile, } from '../utils/fileutils'; -import { NxJsonConfiguration } from '../shared/nx'; +import { logger } from '../utils/logger'; +import { + checkForNPMRC, + detectPackageManager, + getPackageManagerCommand, + resolvePackageVersionUsingRegistry, +} from '../utils/package-manager'; +import { handleErrors } from '../utils/params'; type Dependencies = 'dependencies' | 'devDependencies'; @@ -437,56 +443,207 @@ function createFetcher() { packageName: string, packageVersion: string ): Promise { - if (!cache[`${packageName}-${packageVersion}`]) { - const dir = dirSync().name; - const npmrc = checkForNPMRC(); - if (npmrc) { - // Creating a package.json is needed for .npmrc to resolve - writeJsonFile(`${dir}/package.json`, {}); - // Copy npmrc if it exists, so that npm still follows it. - copyFileSync(npmrc, `${dir}/.npmrc`); - } + if (cache[`${packageName}-${packageVersion}`]) { + return cache[`${packageName}-${packageVersion}`]; + } - logger.info(`Fetching ${packageName}@${packageVersion}`); - const pmc = getPackageManagerCommand(); - execSync(`${pmc.add} ${packageName}@${packageVersion}`, { - stdio: [], - cwd: dir, - }); + let resolvedVersion: string; + let migrations: any; - const migrationsFilePath = packageToMigrationsFilePath(packageName, dir); - const packageJsonPath = require.resolve(`${packageName}/package.json`, { - paths: [dir], - }); - const json = readJsonFile(packageJsonPath); - // packageVersion can be a tag, resolvedVersion works with semver - const resolvedVersion = json.version; - - if (migrationsFilePath) { - const json = readJsonFile(migrationsFilePath); - cache[`${packageName}-${packageVersion}`] = { - version: resolvedVersion, - generators: json.generators || json.schematics, - packageJsonUpdates: json.packageJsonUpdates, - }; - } else { - cache[`${packageName}-${packageVersion}`] = { - version: resolvedVersion, - }; - } + try { + resolvedVersion = resolvePackageVersionUsingRegistry( + packageName, + packageVersion + ); - try { - removeSync(dir); - } catch { - // It's okay if this fails, the OS will clean it up eventually + if (cache[`${packageName}-${resolvedVersion}`]) { + return cache[`${packageName}-${resolvedVersion}`]; } + + logger.info(`Fetching ${packageName}@${packageVersion}`); + migrations = await getPackageMigrations(packageName, resolvedVersion); + } catch { + logger.info(`Fetching ${packageName}@${packageVersion}`); + const result = await installPackageAndGetVersionAngMigrations( + packageName, + packageVersion + ); + resolvedVersion = result.resolvedVersion; + migrations = result.migrations; + } + + if (migrations) { + cache[`${packageName}-${packageVersion}`] = cache[ + `${packageName}-${resolvedVersion}` + ] = { + version: resolvedVersion, + generators: migrations.generators ?? migrations.schematics, + packageJsonUpdates: migrations.packageJsonUpdates, + }; + } else { + cache[`${packageName}-${packageVersion}`] = cache[ + `${packageName}-${resolvedVersion}` + ] = { + version: resolvedVersion, + }; } + return cache[`${packageName}-${packageVersion}`]; }; } // testing-fetch-end +async function getPackageMigrations( + packageName: string, + packageVersion: string +) { + try { + // check if there are migrations in the packages by looking at the + // registry directly + const migrationsPath = getPackageMigrationsPathFromRegistry( + packageName, + packageVersion + ); + if (!migrationsPath) { + return null; + } + + // try to obtain the migrations from the registry directly + return await getPackageMigrationsUsingRegistry( + packageName, + packageVersion, + migrationsPath + ); + } catch { + // fall back to installing the package + const { migrations } = await installPackageAndGetVersionAngMigrations( + packageName, + packageVersion + ); + return migrations; + } +} + +function getPackageMigrationsPathFromRegistry( + packageName: string, + packageVersion: string +): string | null { + let pm = detectPackageManager(); + if (pm === 'yarn') { + pm = 'npm'; + } + const result = execSync( + `${pm} view ${packageName}@${packageVersion} nx-migrations ng-update --json`, + { + stdio: [], + } + ) + .toString() + .trim(); + + if (!result) { + return null; + } + + const json = JSON.parse(result); + let migrationsFilePath = json['nx-migrations'] ?? json['ng-update'] ?? json; + if (typeof json === 'object') { + migrationsFilePath = migrationsFilePath.migrations; + } + + return migrationsFilePath; +} + +async function getPackageMigrationsUsingRegistry( + packageName: string, + packageVersion: string, + migrationsFilePath: string +) { + const dir = dirSync().name; + createNPMRC(dir); + + let pm = detectPackageManager(); + if (pm === 'yarn') { + pm = 'npm'; + } + + const tarballPath = execSync(`${pm} pack ${packageName}@${packageVersion}`, { + cwd: dir, + stdio: [], + }) + .toString() + .trim(); + + let migrations = null; + migrationsFilePath = join('package', migrationsFilePath); + const migrationDestinationPath = join(dir, migrationsFilePath); + try { + await extractFileFromTarball( + join(dir, tarballPath), + migrationsFilePath, + migrationDestinationPath + ); + + migrations = readJsonFile(migrationDestinationPath); + } catch { + throw new Error( + `Failed to find migrations file "${migrationsFilePath}" in package "${packageName}@${packageVersion}".` + ); + } + + try { + removeSync(dir); + } catch { + // It's okay if this fails, the OS will clean it up eventually + } + + return migrations; +} + +async function installPackageAndGetVersionAngMigrations( + packageName: string, + packageVersion: string +) { + const dir = dirSync().name; + createNPMRC(dir); + + const pmc = getPackageManagerCommand(); + execSync(`${pmc.add} ${packageName}@${packageVersion}`, { + stdio: [], + cwd: dir, + }); + + const packageJsonPath = require.resolve(`${packageName}/package.json`, { + paths: [dir], + }); + const { version: resolvedVersion } = readJsonFile(packageJsonPath); + + const migrationsFilePath = packageToMigrationsFilePath(packageName, dir); + let migrations = null; + if (migrationsFilePath) { + migrations = readJsonFile(migrationsFilePath); + } + + try { + removeSync(dir); + } catch { + // It's okay if this fails, the OS will clean it up eventually + } + + return { migrations, resolvedVersion }; +} + +function createNPMRC(dir: string): void { + // A package.json is needed for pnpm pack and for .npmrc to resolve + writeJsonFile(`${dir}/package.json`, {}); + const npmrc = checkForNPMRC(); + if (npmrc) { + // Copy npmrc if it exists, so that npm still follows it. + copyFileSync(npmrc, `${dir}/.npmrc`); + } +} + function packageToMigrationsFilePath(packageName: string, dir: string) { const packageJsonPath = require.resolve(`${packageName}/package.json`, { paths: [dir], @@ -704,16 +861,3 @@ export async function migrate(root: string, args: { [k: string]: any }) { } }); } - -/** - * Checks for a project level npmrc file by crawling up the file tree until - * hitting a package.json file, as this is how npm finds them as well. - */ -function checkForNPMRC(): string | null { - let directory = process.cwd(); - while (!existsSync(join(directory, 'package.json'))) { - directory = dirname(directory); - } - const path = join(directory, '.npmrc'); - return existsSync(path) ? path : null; -} diff --git a/packages/nx/src/utils/fileutils.ts b/packages/nx/src/utils/fileutils.ts index 65d1c31ffb5b2..dc61fff67d405 100644 --- a/packages/nx/src/utils/fileutils.ts +++ b/packages/nx/src/utils/fileutils.ts @@ -1,10 +1,17 @@ import { parseJson, serializeJson } from './json'; import type { JsonParseOptions, JsonSerializeOptions } from './json'; -import { readFileSync, writeFileSync } from 'fs'; +import { + createReadStream, + createWriteStream, + readFileSync, + writeFileSync, +} from 'fs'; import { dirname } from 'path'; import { ensureDirSync } from 'fs-extra'; import { mkdirSync, statSync } from 'fs'; import { resolve as pathResolve } from 'path'; +import * as tar from 'tar-stream'; +import { createGunzip } from 'zlib'; export interface JsonReadOptions extends JsonParseOptions { /** @@ -99,3 +106,47 @@ export function isRelativePath(path: string): boolean { path.startsWith('../') ); } + +/** + * Extracts a file from a given tarball to the specified destination. + * @param tarballPath The path to the tarball from where the file should be extracted. + * @param file The path to the file inside the tarball. + * @param destinationFilePath The destination file path. + * @returns True if the file was extracted successfully, false otherwise. + */ +export async function extractFileFromTarball( + tarballPath: string, + file: string, + destinationFilePath: string +) { + return new Promise((resolve, reject) => { + ensureDirSync(dirname(destinationFilePath)); + var tarExtractStream = tar.extract(); + const destinationFileStream = createWriteStream(destinationFilePath); + + let isFileExtracted = false; + tarExtractStream.on('entry', function (header, stream, next) { + if (header.name === file) { + stream.pipe(destinationFileStream); + stream.on('end', () => { + isFileExtracted = true; + resolve(); + }); + } + + stream.on('end', function () { + next(); + }); + + stream.resume(); + }); + + tarExtractStream.on('finish', function () { + if (!isFileExtracted) { + reject(); + } + }); + + createReadStream(tarballPath).pipe(createGunzip()).pipe(tarExtractStream); + }); +} diff --git a/packages/nx/src/utils/package-manager.ts b/packages/nx/src/utils/package-manager.ts index 692a2988d3867..e9be1f71a4596 100644 --- a/packages/nx/src/utils/package-manager.ts +++ b/packages/nx/src/utils/package-manager.ts @@ -1,6 +1,8 @@ import { execSync } from 'child_process'; -import { existsSync } from 'fs'; -import { join } from 'path'; +import { copyFileSync, existsSync, unlinkSync } from 'fs'; +import { dirname, join } from 'path'; +import { dirSync } from 'tmp'; +import { readJsonFile, writeJsonFile } from './fileutils'; export type PackageManager = 'yarn' | 'pnpm' | 'npm'; @@ -97,3 +99,90 @@ export function getPackageManagerVersion( ): string { return execSync(`${packageManager} --version`).toString('utf-8').trim(); } + +/** + * Checks for a project level npmrc file by crawling up the file tree until + * hitting a package.json file, as this is how npm finds them as well. + */ +export function checkForNPMRC( + directory: string = process.cwd() +): string | null { + while (!existsSync(join(directory, 'package.json'))) { + directory = dirname(directory); + } + const path = join(directory, '.npmrc'); + return existsSync(path) ? path : null; +} + +/** + * Returns the resolved version for a given package and version tag using the + * NPM registry (when using Yarn it will fall back to NPM to fetch the info). + */ +export function resolvePackageVersionUsingRegistry( + packageName: string, + version: string +): string { + let pm = detectPackageManager(); + if (pm === 'yarn') { + pm = 'npm'; + } + + try { + const result = execSync(`${pm} view ${packageName}@${version} version`, { + stdio: [], + }) + .toString() + .trim(); + + if (!result) { + throw new Error(`Unable to resolve version ${packageName}@${version}.`); + } + + // get the last line of the output, strip the package version and quotes + const resolvedVersion = result + .split('\n') + .pop() + .split(' ') + .pop() + .replace(/'/g, ''); + + return resolvedVersion; + } catch { + throw new Error(`Unable to resolve version ${packageName}@${version}.`); + } +} + +/** + * Return the resolved version for a given package and version tag using by + * installing it in a temporary directory and fetching the version from the + * package.json. + */ +export function resolvePackageVersionUsingInstallation( + packageName: string, + version: string +): string { + const dir = dirSync().name; + const npmrc = checkForNPMRC(); + + writeJsonFile(`${dir}/package.json`, {}); + if (npmrc) { + // Copy npmrc if it exists, so that npm still follows it. + copyFileSync(npmrc, `${dir}/.npmrc`); + } + + const pmc = getPackageManagerCommand(); + execSync(`${pmc.add} ${packageName}@${version}`, { stdio: [], cwd: dir }); + + const packageJsonPath = require.resolve(`${packageName}/package.json`, { + paths: [dir], + }); + const { version: resolvedVersion } = readJsonFile(packageJsonPath); + + try { + unlinkSync(dir); + } catch { + // It's okay if this fails, the OS will clean it up eventually + } + + return resolvedVersion; +} diff --git a/yarn.lock b/yarn.lock index 6d4b56850b1da..3df8f91002bc8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -5034,6 +5034,13 @@ resolved "https://registry.yarnpkg.com/@types/tapable/-/tapable-1.0.8.tgz#b94a4391c85666c7b73299fd3ad79d4faa435310" integrity sha512-ipixuVrh2OdNmauvtT51o3d8z12p6LtFW9in7U79der/kwejjdNchQC5UMn5u/KxNoM7VHHOs/l8KS8uHxhODQ== +"@types/tar-stream@^2.2.2": + version "2.2.2" + resolved "https://registry.yarnpkg.com/@types/tar-stream/-/tar-stream-2.2.2.tgz#be9d0be9404166e4b114151f93e8442e6ab6fb1d" + integrity sha512-1AX+Yt3icFuU6kxwmPakaiGrJUwG44MpuiqPg4dSolRFk6jmvs4b3IbUol9wKDLIgU76gevn3EwE8y/DkSJCZQ== + dependencies: + "@types/node" "*" + "@types/tmp@^0.2.0": version "0.2.3" resolved "https://registry.yarnpkg.com/@types/tmp/-/tmp-0.2.3.tgz#908bfb113419fd6a42273674c00994d40902c165" @@ -6899,7 +6906,7 @@ bindings@^1.5.0: dependencies: file-uri-to-path "1.0.0" -bl@^4.1.0: +bl@^4.0.3, bl@^4.1.0: version "4.1.0" resolved "https://registry.yarnpkg.com/bl/-/bl-4.1.0.tgz#451535264182bec2fbbc83a62ab98cf11d9f7b3a" integrity sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w== @@ -9758,7 +9765,7 @@ encoding@^0.1.12, encoding@^0.1.13: dependencies: iconv-lite "^0.6.2" -end-of-stream@^1.0.0, end-of-stream@^1.1.0: +end-of-stream@^1.0.0, end-of-stream@^1.1.0, end-of-stream@^1.4.1: version "1.4.4" resolved "https://registry.yarnpkg.com/end-of-stream/-/end-of-stream-1.4.4.tgz#5ae64a5f45057baf3626ec14da0ca5e4b2431eb0" integrity sha512-+uw1inIHVPQoaVuHzRyXd21icM+cnt4CzD5rW+NC1wjOUSTOs+Te7FOv7AhN7vS9x/oIyhLP5PR1H+phQAHu5Q== @@ -11519,6 +11526,11 @@ fs-access@^1.0.0: dependencies: null-check "^1.0.0" +fs-constants@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/fs-constants/-/fs-constants-1.0.0.tgz#6be0de9be998ce16af8afc24497b9ee9b7ccd9ad" + integrity sha512-y6OAwoSIf7FyjMIv94u+b5rdheZEjzR63GTyZJm5qh4Bi+2YgwLCcI/fPFZkL5PSixOt6ZNKm+w+Hfp/Bciwow== + fs-extra@10.0.1, fs-extra@^10.0.0: version "10.0.1" resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-10.0.1.tgz#27de43b4320e833f6867cc044bfce29fdf0ef3b8" @@ -19433,7 +19445,7 @@ readable-stream@1.1.x: isarray "0.0.1" string_decoder "~0.10.x" -readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.4.0, readable-stream@^3.6.0: +readable-stream@3, readable-stream@^3.0.0, readable-stream@^3.0.6, readable-stream@^3.1.1, readable-stream@^3.4.0, readable-stream@^3.6.0: version "3.6.0" resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.6.0.tgz#337bbda3adc0706bd3e024426a286d4b4b2c9198" integrity sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA== @@ -21544,6 +21556,17 @@ tapable@^2.1.1, tapable@^2.2.0: resolved "https://registry.yarnpkg.com/tapable/-/tapable-2.2.1.tgz#1967a73ef4060a82f12ab96af86d52fdb76eeca0" integrity sha512-GNzQvQTOIP6RyTfE2Qxb8ZVlNmw0n88vp1szwWRimP02mnTsx3Wtn5qRdqY9w2XduFNUgvOwhNnQsjwCp+kqaQ== +tar-stream@~2.2.0: + version "2.2.0" + resolved "https://registry.yarnpkg.com/tar-stream/-/tar-stream-2.2.0.tgz#acad84c284136b060dc3faa64474aa9aebd77287" + integrity sha512-ujeqbceABgwMZxEJnk2HDY2DlnUZ+9oEcb1KzTVfYHio0UE6dG71n60d8D2I4qNvleWrrXpmjpt7vZeF1LnMZQ== + dependencies: + bl "^4.0.3" + end-of-stream "^1.4.1" + fs-constants "^1.0.0" + inherits "^2.0.3" + readable-stream "^3.1.1" + tar@6.1.11, tar@^6.0.2, tar@^6.1.0, tar@^6.1.11, tar@^6.1.2: version "6.1.11" resolved "https://registry.yarnpkg.com/tar/-/tar-6.1.11.tgz#6760a38f003afa1b2ffd0ffe9e9abbd0eab3d621"