Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: download only yarn.js from npm #439

Merged
merged 21 commits into from Apr 1, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 2 additions & 1 deletion config.json
Expand Up @@ -149,7 +149,8 @@
},
"npmRegistry": {
"type": "npm",
"package": "@yarnpkg/cli-dist"
"package": "@yarnpkg/cli-dist",
"bin": "bin/yarn.js"
},
"commands": {
"use": [
Expand Down
103 changes: 75 additions & 28 deletions sources/corepackUtils.ts
Expand Up @@ -15,7 +15,7 @@ import * as httpUtils from './httpUtils
import * as nodeUtils from './nodeUtils';
import * as npmRegistryUtils from './npmRegistryUtils';
import {RegistrySpec, Descriptor, Locator, PackageManagerSpec} from './types';
import {BinList, BinSpec, InstallSpec} from './types';
import {BinList, BinSpec, InstallSpec, DownloadSpec} from './types';

export function getRegistryFromPackageManagerSpec(spec: PackageManagerSpec) {
return process.env.COREPACK_NPM_REGISTRY
Expand Down Expand Up @@ -132,6 +132,66 @@ function isValidBinSpec(x: unknown): x is BinSpec {
return typeof x === `object` && x !== null && !Array.isArray(x) && Object.keys(x).length > 0;
}

async function download(installTarget: string, url: string, algo: string, binPath: string | null = null): Promise<DownloadSpec> {
// Creating a temporary folder inside the install folder means that we
// are sure it'll be in the same drive as the destination, so we can
// just move it there atomically once we are done

const tmpFolder = folderUtils.getTemporaryFolder(installTarget);
debugUtils.log(`Downloading to ${tmpFolder}`);

const stream = await httpUtils.fetchUrlStream(url);

const parsedUrl = new URL(url);
const ext = path.posix.extname(parsedUrl.pathname);

let outputFile: string | null = null;
let sendTo: any;

if (ext === `.tgz`) {
const {default: tar} = await import(`tar`);
sendTo = tar.x({
strip: 1,
cwd: tmpFolder,
filter: binPath ? path => {
const pos = path.indexOf(`/`);
return pos !== -1 && path.slice(pos + 1) === binPath;
} : undefined,
});
} else if (ext === `.js`) {
outputFile = path.join(tmpFolder, path.posix.basename(parsedUrl.pathname));
sendTo = fs.createWriteStream(outputFile);
}
stream.pipe(sendTo);

let hash = !binPath ? stream.pipe(createHash(algo)) : null;
await once(sendTo, `finish`);

if (binPath) {
const downloadedBin = path.join(tmpFolder, binPath);
outputFile = path.join(tmpFolder, path.basename(downloadedBin));
try {
await renameSafe(downloadedBin, outputFile);
} catch (err) {
if ((err as nodeUtils.NodeError)?.code === `ENOENT`)
throw new Error(`Cannot locate '${binPath}' in downloaded tarball`, {cause: err});

throw err;
}

// Calculate the hash of the bin file
const fileStream = fs.createReadStream(outputFile);
hash = fileStream.pipe(createHash(algo));
await once(fileStream, `close`);
}

return {
tmpFolder,
outputFile,
hash: hash!.digest(`hex`),
};
}

export async function installVersion(installTarget: string, locator: Locator, {spec}: {spec: PackageManagerSpec}): Promise<InstallSpec> {
const locatorIsASupportedPackageManager = isSupportedPackageManagerLocator(locator);
const locatorReference = locatorIsASupportedPackageManager ? semver.parse(locator.reference)! : parseURLReference(locator);
Expand Down Expand Up @@ -159,12 +219,16 @@ export async function installVersion(installTarget: string, locator: Locator, {s
}

let url: string;
let binPath: string | null = null;
if (locatorIsASupportedPackageManager) {
url = spec.url.replace(`{}`, version);
if (process.env.COREPACK_NPM_REGISTRY) {
const registry = getRegistryFromPackageManagerSpec(spec);
if (registry.type === `npm`) {
url = await npmRegistryUtils.fetchTarballUrl(registry.package, version);
if (registry.bin) {
binPath = registry.bin;
}
} else {
url = url.replace(
npmRegistryUtils.DEFAULT_NPM_REGISTRY_URL,
Expand All @@ -182,33 +246,9 @@ export async function installVersion(installTarget: string, locator: Locator, {s
}
}

// Creating a temporary folder inside the install folder means that we
// are sure it'll be in the same drive as the destination, so we can
// just move it there atomically once we are done

const tmpFolder = folderUtils.getTemporaryFolder(installTarget);
debugUtils.log(`Installing ${locator.name}@${version} from ${url} to ${tmpFolder}`);
const stream = await httpUtils.fetchUrlStream(url);

const parsedUrl = new URL(url);
const ext = path.posix.extname(parsedUrl.pathname);

let outputFile: string | null = null;

let sendTo: any;
if (ext === `.tgz`) {
const {default: tar} = await import(`tar`);
sendTo = tar.x({strip: 1, cwd: tmpFolder});
} else if (ext === `.js`) {
outputFile = path.join(tmpFolder, path.posix.basename(parsedUrl.pathname));
sendTo = fs.createWriteStream(outputFile);
}

stream.pipe(sendTo);

debugUtils.log(`Installing ${locator.name}@${version} from ${url}`);
const algo = build[0] ?? `sha256`;
const hash = stream.pipe(createHash(algo));
await once(sendTo, `finish`);
const {tmpFolder, outputFile, hash: actualHash} = await download(installTarget, url, algo, binPath);

let bin: BinSpec | BinList;
const isSingleFile = outputFile !== null;
Expand Down Expand Up @@ -240,7 +280,6 @@ export async function installVersion(installTarget: string, locator: Locator, {s
}
}

const actualHash = hash.digest(`hex`);
if (build[1] && actualHash !== build[1])
throw new Error(`Mismatch hashes. Expected ${build[1]}, got ${actualHash}`);

Expand Down Expand Up @@ -305,6 +344,14 @@ export async function installVersion(installTarget: string, locator: Locator, {s
};
}

async function renameSafe(oldPath: fs.PathLike, newPath: fs.PathLike) {
if (process.platform === `win32`) {
await renameUnderWindows(oldPath, newPath);
} else {
await fs.promises.rename(oldPath, newPath);
}
}

async function renameUnderWindows(oldPath: fs.PathLike, newPath: fs.PathLike) {
// Windows malicious file analysis blocks files currently under analysis, so we need to wait for file release
const retries = 5;
Expand Down
7 changes: 7 additions & 0 deletions sources/types.ts
Expand Up @@ -25,6 +25,7 @@ export function isSupportedPackageManager(value: string): value is SupportedPack
export interface NpmRegistrySpec {
type: `npm`;
package: string;
bin?: string;
}

export interface UrlRegistrySpec {
Expand Down Expand Up @@ -59,6 +60,12 @@ export interface InstallSpec {
hash: string;
}

export interface DownloadSpec {
tmpFolder: string;
outputFile: string | null;
hash: string;
}

/**
* The data structure found in config.json
*/
Expand Down
8 changes: 4 additions & 4 deletions tests/main.test.ts
Expand Up @@ -787,19 +787,19 @@ it(`should download yarn berry from custom registry`, async () => {
process.env.COREPACK_ENABLE_DOWNLOAD_PROMPT = `1`;

await xfs.writeJsonPromise(ppath.join(cwd, `package.json` as Filename), {
packageManager: `yarn@3.0.0`,
packageManager: `yarn@3.0.0-rc.2+sha224.f83f6d1cbfac10ba6b516a62ccd2a72ccd857aa6c514d1cd7185ec60`,
});

await expect(runCli(cwd, [`yarn`, `--version`])).resolves.toMatchObject({
exitCode: 0,
stdout: `3.0.0\n`,
stderr: `! Corepack is about to download https://registry.npmmirror.com/@yarnpkg/cli-dist/-/cli-dist-3.0.0.tgz\n`,
stdout: `3.0.0-rc.2\n`,
stderr: `! Corepack is about to download https://registry.npmmirror.com/@yarnpkg/cli-dist/-/cli-dist-3.0.0-rc.2.tgz\n`,
});

// Should keep working with cache
await expect(runCli(cwd, [`yarn`, `--version`])).resolves.toMatchObject({
exitCode: 0,
stdout: `3.0.0\n`,
stdout: `3.0.0-rc.2\n`,
stderr: ``,
});
});
Expand Down
Binary file modified tests/nocks.db
Binary file not shown.