From c9c54fe9dd5f519a73ecf62b5c484eaec5761d92 Mon Sep 17 00:00:00 2001 From: Kenrick Date: Mon, 11 Apr 2022 04:04:49 +0000 Subject: [PATCH] feat(git-fetcher): shallow clone when fetching git resource --- .changeset/curly-spiders-search.md | 5 ++ packages/client/src/index.ts | 2 + packages/config/src/Config.ts | 1 + packages/config/src/index.ts | 9 ++++ packages/git-fetcher/src/index.ts | 25 +++++++++- packages/git-fetcher/test/index.ts | 47 +++++++++++++++++++ .../src/createNewStoreController.ts | 2 + 7 files changed, 90 insertions(+), 1 deletion(-) create mode 100644 .changeset/curly-spiders-search.md diff --git a/.changeset/curly-spiders-search.md b/.changeset/curly-spiders-search.md new file mode 100644 index 00000000000..8e0681923c4 --- /dev/null +++ b/.changeset/curly-spiders-search.md @@ -0,0 +1,5 @@ +--- +"@pnpm/git-fetcher": minor +--- + +feat(git-fetcher): shallow clone when fetching git resource diff --git a/packages/client/src/index.ts b/packages/client/src/index.ts index 1b6fc93bc44..05ca9cc15a3 100644 --- a/packages/client/src/index.ts +++ b/packages/client/src/index.ts @@ -18,6 +18,7 @@ export type ClientOptions = { timeout?: number userAgent?: string userConfig?: Record + gitShallowHosts?: string[] } & ResolverFactoryOptions & AgentOptions export default function (opts: ClientOptions) { @@ -40,6 +41,7 @@ function createFetchers ( getCredentials: GetCredentials, opts: { retry?: RetryTimeoutOptions + gitShallowHosts?: string[] } ) { return { diff --git a/packages/config/src/Config.ts b/packages/config/src/Config.ts index 9805bc5e7d9..cc73d0c550c 100644 --- a/packages/config/src/Config.ts +++ b/packages/config/src/Config.ts @@ -140,6 +140,7 @@ export interface Config { enableModulesDir: boolean modulesCacheMaxAge: number embedReadme?: boolean + gitShallowHosts?: string[] registries: Registries ignoreWorkspaceRootCheck: boolean diff --git a/packages/config/src/index.ts b/packages/config/src/index.ts index 22677ecbd7f..982ba55c85e 100644 --- a/packages/config/src/index.ts +++ b/packages/config/src/index.ts @@ -48,6 +48,7 @@ export const types = Object.assign({ 'filter-prod': [String, Array], 'frozen-lockfile': Boolean, 'git-checks': Boolean, + 'git-shallow-hosts': Array, 'global-bin-dir': String, 'global-dir': String, 'global-path': String, @@ -174,6 +175,14 @@ export default async ( 'fetch-retry-maxtimeout': 60000, 'fetch-retry-mintimeout': 10000, 'fetch-timeout': 60000, + 'git-shallow-hosts': [ + // Follow https://github.com/npm/git/blob/1e1dbd26bd5b87ca055defecc3679777cb480e2a/lib/clone.js#L13-L19 + 'github.com', + 'gist.github.com', + 'gitlab.com', + 'bitbucket.com', + 'bitbucket.org', + ], globalconfig: npmDefaults.globalconfig, hoist: true, 'hoist-pattern': ['*'], diff --git a/packages/git-fetcher/src/index.ts b/packages/git-fetcher/src/index.ts index b5f9bf9d721..3170fd87576 100644 --- a/packages/git-fetcher/src/index.ts +++ b/packages/git-fetcher/src/index.ts @@ -3,6 +3,7 @@ import { Cafs, DeferredManifestPromise } from '@pnpm/fetcher-base' import preparePackage from '@pnpm/prepare-package' import rimraf from '@zkochan/rimraf' import execa from 'execa' +import { URL } from 'url' export default () => { return { @@ -15,10 +16,17 @@ export default () => { }, opts: { manifest?: DeferredManifestPromise + gitShallowHosts?: string[] } ) { const tempLocation = await cafs.tempDir() - await execGit(['clone', resolution.repo, tempLocation]) + if (shouldUseShallow(resolution.repo, opts.gitShallowHosts)) { + await execGit(['init'], { cwd: tempLocation }) + await execGit(['remote', 'add', 'origin', resolution.repo], { cwd: tempLocation }) + await execGit(['fetch', '--depth', '1', 'origin', resolution.commit], { cwd: tempLocation }) + } else { + await execGit(['clone', resolution.repo, tempLocation]) + } await execGit(['checkout', resolution.commit], { cwd: tempLocation }) await preparePackage(tempLocation) // removing /.git to make directory integrity calculation faster @@ -32,6 +40,21 @@ export default () => { } } +function shouldUseShallow (repoUrl: string, allowedHosts: string[] = []): boolean { + if (!allowedHosts) { + return false + } + try { + const { host } = new URL(repoUrl) + if (allowedHosts.includes(host)) { + return true + } + } catch (e) { + // URL might be malformed + } + return false +} + function prefixGitArgs (): string[] { return process.platform === 'win32' ? ['-c', 'core.longpaths=true'] : [] } diff --git a/packages/git-fetcher/test/index.ts b/packages/git-fetcher/test/index.ts index 8c5589be420..17dcc46817f 100644 --- a/packages/git-fetcher/test/index.ts +++ b/packages/git-fetcher/test/index.ts @@ -5,6 +5,20 @@ import createFetcher from '@pnpm/git-fetcher' import { DependencyManifest } from '@pnpm/types' import pDefer from 'p-defer' import tempy from 'tempy' +import execa from 'execa' + +jest.mock('execa', () => { + const originalModule = jest.requireActual('execa') + return { + __esModule: true, + ...originalModule, + default: jest.fn(originalModule.default), + } +}) + +beforeEach(() => { + (execa as jest.Mock).mockClear() +}) test('fetch', async () => { const cafsDir = tempy.directory() @@ -78,3 +92,36 @@ test('fetch a big repository', async () => { }, { manifest }) await Promise.all(Object.values(filesIndex).map(({ writeResult }) => writeResult)) }) + +test('still able to shallow fetch for allowed hosts', async () => { + const cafsDir = tempy.directory() + const fetch = createFetcher().git + const manifest = pDefer() + const resolution = { + commit: 'c9b30e71d704cd30fa71f2edd1ecc7dcc4985493', + repo: 'https://github.com/kevva/is-positive.git', + type: 'git' as const, + } + const { filesIndex } = await fetch( + createCafsStore(cafsDir), + resolution, + { + manifest, + gitShallowHosts: ['github.com'], + } + ) + const calls = (execa as jest.Mock).mock.calls + const expectedCalls = [ + ['git', ['init']], + ['git', ['remote', 'add', 'origin', resolution.repo]], + ['git', ['fetch', '--depth', '1', 'origin', resolution.commit]], + ] + for (let i = 0; i < expectedCalls.length; i++) { + // Discard final argument as it passes temporary directory + expect(calls[i].slice(0, -1)).toEqual(expectedCalls[i]) + } + expect(filesIndex['package.json']).toBeTruthy() + expect(filesIndex['package.json'].writeResult).toBeTruthy() + const name = (await manifest.promise).name + expect(name).toEqual('is-positive') +}) \ No newline at end of file diff --git a/packages/store-connection-manager/src/createNewStoreController.ts b/packages/store-connection-manager/src/createNewStoreController.ts index e030ec64e30..955973966e7 100644 --- a/packages/store-connection-manager/src/createNewStoreController.ts +++ b/packages/store-connection-manager/src/createNewStoreController.ts @@ -22,6 +22,7 @@ export type CreateNewStoreControllerOptions = CreateResolverOptions & Pick