Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat(git): blobless git (#11401)
  • Loading branch information
rarkins committed Sep 9, 2021
1 parent 367c901 commit d6d4124
Show file tree
Hide file tree
Showing 5 changed files with 22 additions and 63 deletions.
2 changes: 1 addition & 1 deletion docs/development/local-development.md
Expand Up @@ -10,7 +10,7 @@ For example, if you think anything is unclear, or you think something needs to b

You need the following dependencies for local development:

- Git `>=2.22.0`
- Git `>=2.33.0`
- Node.js `>=14.15.4`
- Yarn `^1.22.5`
- C++ compiler
Expand Down
2 changes: 2 additions & 0 deletions docs/usage/getting-started/running.md
Expand Up @@ -27,6 +27,8 @@ Therefore if you need Renovate to support any non-npm lock files like Bundler th

The `renovate` npm package is compatible with all of Renovate's supported platforms.

Renovate requires Node.js >=14.15.0 as well as Git >=2.33.0.

#### Docker image

The `renovate` npm package is also distributed via pre-built Node.js images on Docker Hub (`renovate/renovate`).
Expand Down
8 changes: 4 additions & 4 deletions lib/util/git/index.spec.ts
Expand Up @@ -75,7 +75,7 @@ describe('util/git/index', () => {
await git.initRepo({
url: origin.path,
});
await git.setUserRepoConfig({ branchPrefix: 'renovate/' });
git.setUserRepoConfig({ branchPrefix: 'renovate/' });
git.setGitAuthor('Jest <Jest@example.com>');
setNoVerify([]);
await git.syncGit();
Expand Down Expand Up @@ -159,7 +159,7 @@ describe('util/git/index', () => {
expect(await git.isBranchModified('renovate/future_branch')).toBe(false);
});
it('should return false when author is ignored', async () => {
await git.setUserRepoConfig({
git.setUserRepoConfig({
gitIgnoredAuthors: ['custom@example.com'],
});
expect(await git.isBranchModified('renovate/custom_author')).toBe(false);
Expand Down Expand Up @@ -494,7 +494,7 @@ describe('util/git/index', () => {
url: base.path,
});

await git.setUserRepoConfig({ branchPrefix: 'renovate/' });
git.setUserRepoConfig({ branchPrefix: 'renovate/' });
expect(git.branchExists('renovate/test')).toBe(true);

await git.initRepo({
Expand All @@ -504,7 +504,7 @@ describe('util/git/index', () => {
await repo.checkout('renovate/test');
await repo.commit('past message3', ['--amend']);

await git.setUserRepoConfig({ branchPrefix: 'renovate/' });
git.setUserRepoConfig({ branchPrefix: 'renovate/' });
expect(git.branchExists('renovate/test')).toBe(true);
});

Expand Down
71 changes: 14 additions & 57 deletions lib/util/git/index.ts
Expand Up @@ -53,7 +53,6 @@ interface LocalConfig extends StorageConfig {
currentBranchSha: string;
branchCommits: Record<string, CommitSha>;
branchIsModified: Record<string, boolean>;
branchPrefix: string;
ignoredAuthors: string[];
gitAuthorName?: string;
gitAuthorEmail?: string;
Expand Down Expand Up @@ -215,25 +214,6 @@ async function cleanLocalBranches(): Promise<void> {
}
}

/*
* When we initially clone, we clone only the default branch so how no knowledge of other branches existing.
* By calling this function once the repo's branchPrefix is known, we can fetch all of Renovate's branches in one command.
*/
async function setBranchPrefix(branchPrefix: string): Promise<void> {
config.branchPrefix = branchPrefix;
// If the repo is already cloned then set branchPrefix now, otherwise it will be called again during syncGit()
if (gitInitialized) {
logger.debug('Setting branchPrefix: ' + branchPrefix);
const ref = `refs/heads/${branchPrefix}*:refs/remotes/origin/${branchPrefix}*`;
try {
await git.fetch(['origin', ref, '--depth=5', '--force']);
} catch (err) /* istanbul ignore next */ {
checkForPlatformFailure(err);
throw err;
}
}
}

export function setGitAuthor(gitAuthor: string): void {
const gitAuthorParsed = parseGitAuthor(
gitAuthor || 'Renovate Bot <renovate@whitesourcesoftware.com>'
Expand Down Expand Up @@ -270,12 +250,10 @@ export async function writeGitAuthor(): Promise<void> {
}
}

export async function setUserRepoConfig({
branchPrefix,
export function setUserRepoConfig({
gitIgnoredAuthors,
gitAuthor,
}: RenovateConfig): Promise<void> {
await setBranchPrefix(branchPrefix);
}: RenovateConfig): void {
config.ignoredAuthors = gitIgnoredAuthors ?? [];
setGitAuthor(gitAuthor);
}
Expand Down Expand Up @@ -316,7 +294,7 @@ export async function syncGit(): Promise<void> {
await resetToBranch(await getDefaultBranch(git));
const fetchStart = Date.now();
await git.pull();
await git.fetch(['--depth=10']);

This comment has been minimized.

Copy link
@ericcitaire

ericcitaire Sep 15, 2021

Contributor

Removing this option here and line 340 makes Git Submodules manager Renovate fail on a Bitbucket Server platform with this error :

Logs
 WARN: Host error (repository=prj/repo)
       "hostType": "git",
       "lookupName": undefined,
       "err": {
         "task": {
           "commands": [
             "clone",
             "--filter=blob:none",
             "https://****:**redacted**@bitbucket.acme.com/scm/prj/repo.git",
             "."
           ],
           "format": "utf-8"
         },
         "message": "Cloning into '.'...\nwarning: filtering not recognized by server, ignoring\nerror: RPC failed; curl 56 GnuTLS recv error (-9): Error decoding the received TLS packet.\nerror: 6653 bytes of body are still expected\nfetch-pack: unexpected disconnect while reading sideband packet\nfatal: early EOF\nfatal: fetch-pack: invalid index-pack output\n",
         "stack": "Error: Cloning into '.'...\nwarning: filtering not recognized by server, ignoring\nerror: RPC failed; curl 56 GnuTLS recv error (-9): Error decoding the received TLS packet.\nerror: 6653 bytes of body are still expected\nfetch-pack: unexpected disconnect while reading sideband packet\nfatal: early EOF\nfatal: fetch-pack: invalid index-pack output\n\n    at Object.action (/usr/src/app/node_modules/simple-git/src/lib/plugins/error-detection.plugin.ts:38:28)\n    at PluginStore.exec (/usr/src/app/node_modules/simple-git/src/lib/plugins/plugin-store.ts:24:29)\n    at /usr/src/app/node_modules/simple-git/src/lib/runners/git-executor-chain.ts:114:40\n    at new Promise (<anonymous>)\n    at GitExecutorChain.handleTaskData (/usr/src/app/node_modules/simple-git/src/lib/runners/git-executor-chain.ts:111:14)\n    at GitExecutorChain.<anonymous> (/usr/src/app/node_modules/simple-git/src/lib/runners/git-executor-chain.ts:88:40)\n    at Generator.next (<anonymous>)\n    at fulfilled (/usr/src/app/node_modules/simple-git/src/lib/runners/git-executor-chain.js:5:58)\n    at processTicksAndRejections (internal/process/task_queues.js:95:5)"
       }

Adding back --depth=10 fixes it.

This comment has been minimized.

Copy link
@viceice

viceice Sep 15, 2021

Member

please open a discussion for this

await git.fetch();
config.currentBranch =
config.currentBranch || (await getDefaultBranch(git));
await resetToBranch(config.currentBranch);
Expand All @@ -336,8 +314,8 @@ export async function syncGit(): Promise<void> {
await fs.emptyDir(localDir);
const cloneStart = Date.now();
try {
// clone only the default branch
const opts = ['--depth=10'];
// blobless clone
const opts = ['--filter=blob:none'];
if (config.extraCloneOpts) {
Object.entries(config.extraCloneOpts).forEach((e) =>
opts.push(e[0], `${e[1]}`)
Expand Down Expand Up @@ -383,9 +361,6 @@ export async function syncGit(): Promise<void> {
logger.warn({ err }, 'Cannot retrieve latest commit');
}
config.currentBranch = config.currentBranch || (await getDefaultBranch(git));
if (config.branchPrefix) {
await setBranchPrefix(config.branchPrefix);
}
}

// istanbul ignore next
Expand All @@ -394,24 +369,6 @@ export async function getRepoStatus(): Promise<StatusResult> {
return git.status();
}

async function syncBranch(branchName: string): Promise<void> {
await syncGit();
if (branchName.startsWith(config.branchPrefix)) {
return;
}
if (config.additionalBranches.includes(branchName)) {
return;
}
config.additionalBranches.push(branchName);
// fetch the branch only if it's not part of the existing branchPrefix
try {
await git.raw(['remote', 'set-branches', '--add', 'origin', branchName]);
await git.fetch(['origin', branchName, '--depth=5']);
} catch (err) /* istanbul ignore next */ {
checkForPlatformFailure(err);
}
}

export function branchExists(branchName: string): boolean {
return !!config.branchCommits[branchName];
}
Expand Down Expand Up @@ -447,7 +404,7 @@ export async function getCommitMessages(): Promise<string[]> {

export async function checkoutBranch(branchName: string): Promise<CommitSha> {
logger.debug(`Setting current branch to ${branchName}`);
await syncBranch(branchName);
await syncGit();
try {
config.currentBranch = branchName;
config.currentBranchSha = (
Expand Down Expand Up @@ -506,7 +463,7 @@ export function getBranchList(): string[] {
}

export async function isBranchStale(branchName: string): Promise<boolean> {
await syncBranch(branchName);
await syncGit();
try {
const { currentBranchSha, currentBranch } = config;
const branches = await git.branch([
Expand All @@ -528,7 +485,7 @@ export async function isBranchStale(branchName: string): Promise<boolean> {
}

export async function isBranchModified(branchName: string): Promise<boolean> {
await syncBranch(branchName);
await syncGit();
// First check cache
if (config.branchIsModified[branchName] !== undefined) {
return config.branchIsModified[branchName];
Expand Down Expand Up @@ -580,7 +537,7 @@ export async function isBranchModified(branchName: string): Promise<boolean> {
}

export async function deleteBranch(branchName: string): Promise<void> {
await syncBranch(branchName);
await syncGit();
try {
await git.raw(['push', '--delete', 'origin', branchName]);
logger.debug({ branchName }, 'Deleted remote branch');
Expand All @@ -602,7 +559,7 @@ export async function deleteBranch(branchName: string): Promise<void> {
export async function mergeBranch(branchName: string): Promise<void> {
let status;
try {
await syncBranch(branchName);
await syncGit();
await git.reset(ResetMode.HARD);
await git.checkout(['-B', branchName, 'origin/' + branchName]);
await git.checkout([
Expand Down Expand Up @@ -633,7 +590,7 @@ export async function mergeBranch(branchName: string): Promise<void> {
export async function getBranchLastCommitTime(
branchName: string
): Promise<Date> {
await syncBranch(branchName);
await syncGit();
try {
const time = await git.show(['-s', '--format=%ai', 'origin/' + branchName]);
return new Date(Date.parse(time));
Expand All @@ -644,7 +601,7 @@ export async function getBranchLastCommitTime(
}

export async function getBranchFiles(branchName: string): Promise<string[]> {
await syncBranch(branchName);
await syncGit();
try {
const diff = await git.diffSummary([
`origin/${branchName}`,
Expand Down Expand Up @@ -675,7 +632,7 @@ export async function getFile(
}

export async function hasDiff(branchName: string): Promise<boolean> {
await syncBranch(branchName);
await syncGit();
try {
return (await git.diff(['HEAD', branchName])) !== '';
} catch (err) {
Expand Down Expand Up @@ -816,7 +773,7 @@ export async function commitFiles({
logger.debug({ result: pushRes }, 'git push');
// Fetch it after create
const ref = `refs/heads/${branchName}:refs/remotes/origin/${branchName}`;
await git.fetch(['origin', ref, '--depth=5', '--force']);
await git.fetch(['origin', ref, '--force']);
config.branchCommits[branchName] = (
await git.revparse([branchName])
).trim();
Expand Down
2 changes: 1 addition & 1 deletion lib/workers/repository/init/index.ts
Expand Up @@ -32,7 +32,7 @@ export async function initRepo(
checkIfConfigured(config);
warnOnUnsupportedOptions(config);
config = applySecretsToConfig(config);
await setUserRepoConfig(config);
setUserRepoConfig(config);
config = await detectVulnerabilityAlerts(config);
// istanbul ignore if
if (config.printConfig) {
Expand Down

0 comments on commit d6d4124

Please sign in to comment.