From d144013af1515bb51efa3978a483847feac39f35 Mon Sep 17 00:00:00 2001 From: Sergio Zharinov Date: Wed, 1 Jul 2020 14:45:53 +0400 Subject: [PATCH] refactor(platform): Make git to be class-less (#6635) Co-authored-by: Michael Kriese Co-authored-by: Rhys Arkins --- lib/manager/composer/artifacts.spec.ts | 2 +- lib/manager/gomod/artifacts.spec.ts | 2 +- lib/manager/pipenv/artifacts.spec.ts | 2 +- lib/platform/azure/index.spec.ts | 25 +- lib/platform/azure/index.ts | 43 +- lib/platform/bitbucket-server/index.spec.ts | 45 +- lib/platform/bitbucket-server/index.ts | 47 +- lib/platform/bitbucket-server/types.ts | 2 - lib/platform/bitbucket/index.spec.ts | 26 +- lib/platform/bitbucket/index.ts | 42 +- lib/platform/bitbucket/utils.ts | 2 - lib/platform/git/index.ts | 600 ------------------ lib/platform/gitea/index.spec.ts | 107 ++-- lib/platform/gitea/index.ts | 42 +- lib/platform/github/index.spec.ts | 32 +- lib/platform/github/index.ts | 44 +- lib/platform/github/types.ts | 2 - lib/platform/gitlab/index.spec.ts | 37 +- lib/platform/gitlab/index.ts | 47 +- lib/platform/index.ts | 2 +- .../git/__snapshots__/index.spec.ts.snap | 0 .../gitfs}/git/index.spec.ts | 133 ++-- lib/util/gitfs/git/index.ts | 599 +++++++++++++++++ .../gitfs}/git/private-key.spec.ts | 6 +- .../gitfs}/git/private-key.ts | 6 +- lib/util/gitfs/index.ts | 2 + lib/workers/branch/index.spec.ts | 2 +- 27 files changed, 877 insertions(+), 1022 deletions(-) delete mode 100644 lib/platform/git/index.ts rename lib/{platform => util/gitfs}/git/__snapshots__/index.spec.ts.snap (100%) rename lib/{platform => util/gitfs}/git/index.spec.ts (74%) create mode 100644 lib/util/gitfs/git/index.ts rename lib/{platform => util/gitfs}/git/private-key.spec.ts (89%) rename lib/{platform => util/gitfs}/git/private-key.ts (88%) diff --git a/lib/manager/composer/artifacts.spec.ts b/lib/manager/composer/artifacts.spec.ts index 1488b1ee350f76..aebbc6b046095f 100644 --- a/lib/manager/composer/artifacts.spec.ts +++ b/lib/manager/composer/artifacts.spec.ts @@ -2,11 +2,11 @@ import { exec as _exec } from 'child_process'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../test/execUtil'; import { mocked, platform } from '../../../test/util'; -import { StatusResult } from '../../platform/git'; import { setUtilConfig } from '../../util'; import { BinarySource } from '../../util/exec/common'; import * as docker from '../../util/exec/docker'; import * as _env from '../../util/exec/env'; +import { StatusResult } from '../../util/gitfs'; import * as _gitfs from '../../util/gitfs'; import * as composer from './artifacts'; diff --git a/lib/manager/gomod/artifacts.spec.ts b/lib/manager/gomod/artifacts.spec.ts index dec48a418b69f4..3bb563100d6e83 100644 --- a/lib/manager/gomod/artifacts.spec.ts +++ b/lib/manager/gomod/artifacts.spec.ts @@ -3,11 +3,11 @@ import _fs from 'fs-extra'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../test/execUtil'; import { mocked, platform } from '../../../test/util'; -import { StatusResult } from '../../platform/git'; import { setUtilConfig } from '../../util'; import { BinarySource } from '../../util/exec/common'; import * as docker from '../../util/exec/docker'; import * as _env from '../../util/exec/env'; +import { StatusResult } from '../../util/gitfs'; import * as _hostRules from '../../util/host-rules'; import * as gomod from './artifacts'; diff --git a/lib/manager/pipenv/artifacts.spec.ts b/lib/manager/pipenv/artifacts.spec.ts index 084f94254b66fb..9e6f5a42c6b7a5 100644 --- a/lib/manager/pipenv/artifacts.spec.ts +++ b/lib/manager/pipenv/artifacts.spec.ts @@ -3,11 +3,11 @@ import _fs from 'fs-extra'; import { join } from 'upath'; import { envMock, mockExecAll } from '../../../test/execUtil'; import { mocked, platform } from '../../../test/util'; -import { StatusResult } from '../../platform/git'; import { setUtilConfig } from '../../util'; import { BinarySource } from '../../util/exec/common'; import * as docker from '../../util/exec/docker'; import * as _env from '../../util/exec/env'; +import { StatusResult } from '../../util/gitfs'; import * as pipenv from './artifacts'; jest.mock('fs-extra'); diff --git a/lib/platform/azure/index.spec.ts b/lib/platform/azure/index.spec.ts index 16aba35f38bdcd..fed43031377b86 100644 --- a/lib/platform/azure/index.spec.ts +++ b/lib/platform/azure/index.spec.ts @@ -1,6 +1,7 @@ import is from '@sindresorhus/is'; import { REPOSITORY_DISABLED } from '../../constants/error-messages'; import { BranchStatus } from '../../types'; +import * as _gitfs from '../../util/gitfs'; import * as _hostRules from '../../util/host-rules'; import { Platform, RepoParams } from '../common'; @@ -9,36 +10,22 @@ describe('platform/azure', () => { let azure: Platform; let azureApi: jest.Mocked; let azureHelper: jest.Mocked; - let GitStorage; + let gitfs: jest.Mocked; beforeEach(async () => { // reset module jest.resetModules(); jest.mock('./azure-got-wrapper'); jest.mock('./azure-helper'); - jest.mock('../git'); + jest.mock('../../util/gitfs'); jest.mock('../../util/host-rules'); hostRules = require('../../util/host-rules'); require('../../util/sanitize').sanitize = jest.fn((input) => input); azure = await import('.'); azureApi = require('./azure-got-wrapper'); azureHelper = require('./azure-helper'); - GitStorage = require('../git').Storage; - GitStorage.mockImplementation(() => ({ - initRepo: jest.fn(), - cleanRepo: jest.fn(), - getFileList: jest.fn(), - branchExists: jest.fn(() => true), - isBranchStale: jest.fn(() => false), - setBaseBranch: jest.fn(), - getBranchLastCommitTime: jest.fn(), - getAllRenovateBranches: jest.fn(), - getCommitMessages: jest.fn(), - getFile: jest.fn(), - commitFiles: jest.fn(), - mergeBranch: jest.fn(), - deleteBranch: jest.fn(), - getRepoStatus: jest.fn(), - })); + gitfs = require('../../util/gitfs'); + gitfs.branchExists.mockResolvedValue(true); + gitfs.isBranchStale.mockResolvedValue(false); hostRules.find.mockReturnValue({ token: 'token', }); diff --git a/lib/platform/azure/index.ts b/lib/platform/azure/index.ts index af4f87ec9f1563..399e0f294dcad7 100644 --- a/lib/platform/azure/index.ts +++ b/lib/platform/azure/index.ts @@ -13,6 +13,7 @@ import { } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; +import * as gitfs from '../../util/gitfs'; import * as hostRules from '../../util/host-rules'; import { sanitize } from '../../util/sanitize'; import { ensureTrailingSlash } from '../../util/url'; @@ -31,14 +32,12 @@ import { RepoParams, VulnerabilityAlert, } from '../common'; -import GitStorage, { StatusResult } from '../git'; import { smartTruncate } from '../utils/pr-body'; import * as azureApi from './azure-got-wrapper'; import * as azureHelper from './azure-helper'; import { AzurePr } from './types'; interface Config { - storage: GitStorage; repoForceRebase: boolean; mergeMethod: GitPullRequestMergeStrategy; baseCommitSHA: string | undefined; @@ -139,6 +138,7 @@ export async function initRepo({ interface RenovateConfig { enabled: boolean; } + let renovateConfig: RenovateConfig; try { const json = await azureHelper.getFile( @@ -155,7 +155,6 @@ export async function initRepo({ } } - config.storage = new GitStorage(); const [projectName, repoName] = repository.split('/'); const opts = hostRules.find({ hostType: defaults.hostType, @@ -164,7 +163,7 @@ export async function initRepo({ const url = defaults.endpoint + `${encodeURIComponent(projectName)}/_git/${encodeURIComponent(repoName)}`; - await config.storage.initRepo({ + await gitfs.initRepo({ ...config, localDir, url, @@ -186,7 +185,7 @@ export function getRepoForceRebase(): Promise { // Search export /* istanbul ignore next */ function getFileList(): Promise { - return config.storage.getFileList(); + return gitfs.getFileList(); } export /* istanbul ignore next */ async function setBaseBranch( @@ -195,14 +194,14 @@ export /* istanbul ignore next */ async function setBaseBranch( logger.debug(`Setting baseBranch to ${branchName}`); config.baseBranch = branchName; delete config.baseCommitSHA; - const baseBranchSha = await config.storage.setBaseBranch(branchName); + const baseBranchSha = await gitfs.setBaseBranch(branchName); return baseBranchSha; } export /* istanbul ignore next */ function setBranchPrefix( branchPrefix: string ): Promise { - return config.storage.setBranchPrefix(branchPrefix); + return gitfs.setBranchPrefix(branchPrefix); } // Branch @@ -210,26 +209,26 @@ export /* istanbul ignore next */ function setBranchPrefix( export /* istanbul ignore next */ function branchExists( branchName: string ): Promise { - return config.storage.branchExists(branchName); + return gitfs.branchExists(branchName); } export /* istanbul ignore next */ function getAllRenovateBranches( branchPrefix: string ): Promise { - return config.storage.getAllRenovateBranches(branchPrefix); + return gitfs.getAllRenovateBranches(branchPrefix); } export /* istanbul ignore next */ function isBranchStale( branchName: string ): Promise { - return config.storage.isBranchStale(branchName); + return gitfs.isBranchStale(branchName); } export /* istanbul ignore next */ function getFile( filePath: string, branchName: string ): Promise { - return config.storage.getFile(filePath, branchName); + return gitfs.getFile(filePath, branchName); } // istanbul ignore next @@ -273,7 +272,7 @@ export async function getPrList(): Promise { /* istanbul ignore next */ export async function getPrFiles(pr: Pr): Promise { - return config.storage.getBranchFiles(pr.branchName, pr.targetBranch); + return gitfs.getBranchFiles(pr.branchName, pr.targetBranch); } export async function getPr(pullRequestId: number): Promise { @@ -309,6 +308,7 @@ export async function getPr(pullRequestId: number): Promise { return azurePr; } + export async function findPr({ branchName, prTitle, @@ -361,7 +361,7 @@ export /* istanbul ignore next */ async function deleteBranch( branchName: string, abandonAssociatedPr = false ): Promise { - await config.storage.deleteBranch(branchName); + await gitfs.deleteBranch(branchName); if (abandonAssociatedPr) { const pr = await getBranchPr(branchName); await abandonPr(pr.number); @@ -371,19 +371,19 @@ export /* istanbul ignore next */ async function deleteBranch( export /* istanbul ignore next */ function getBranchLastCommitTime( branchName: string ): Promise { - return config.storage.getBranchLastCommitTime(branchName); + return gitfs.getBranchLastCommitTime(branchName); } export /* istanbul ignore next */ function getRepoStatus(): Promise< - StatusResult + gitfs.StatusResult > { - return config.storage.getRepoStatus(); + return gitfs.getRepoStatus(); } export /* istanbul ignore next */ function mergeBranch( branchName: string ): Promise { - return config.storage.mergeBranch(branchName); + return gitfs.mergeBranch(branchName); } export /* istanbul ignore next */ function commitFiles({ @@ -391,7 +391,7 @@ export /* istanbul ignore next */ function commitFiles({ files, message, }: CommitFilesConfig): Promise { - return config.storage.commitFiles({ + return gitfs.commitFiles({ branchName, files, message, @@ -401,7 +401,7 @@ export /* istanbul ignore next */ function commitFiles({ export /* istanbul ignore next */ function getCommitMessages(): Promise< string[] > { - return config.storage.getCommitMessages(); + return gitfs.getCommitMessages(); } export async function getBranchStatusCheck( @@ -769,10 +769,7 @@ export function getVulnerabilityAlerts(): Promise { } export function cleanRepo(): Promise { - // istanbul ignore if - if (config.storage && config.storage.cleanRepo) { - config.storage.cleanRepo(); - } + gitfs.cleanRepo(); config = {} as any; return Promise.resolve(); } diff --git a/lib/platform/bitbucket-server/index.spec.ts b/lib/platform/bitbucket-server/index.spec.ts index 9b147416b53a13..7b1f9f589656c3 100644 --- a/lib/platform/bitbucket-server/index.spec.ts +++ b/lib/platform/bitbucket-server/index.spec.ts @@ -7,8 +7,8 @@ import { } from '../../constants/error-messages'; import { PR_STATE_CLOSED, PR_STATE_OPEN } from '../../constants/pull-requests'; import { BranchStatus } from '../../types'; +import * as _gitfs from '../../util/gitfs'; import { Platform } from '../common'; -import { Storage } from '../git'; function repoMock( endpoint: URL | string, @@ -143,9 +143,7 @@ describe('platform/bitbucket-server', () => { describe(scenarioName, () => { let bitbucket: Platform; let hostRules: jest.Mocked; - let GitStorage: jest.Mock & { - getUrl: jest.MockInstance; - }; + let gitfs: jest.Mocked; async function initRepo(config = {}): Promise { const scope = httpMock @@ -174,32 +172,15 @@ describe('platform/bitbucket-server', () => { httpMock.reset(); httpMock.setup(); jest.mock('delay'); - jest.mock('../git'); + jest.mock('../../util/gitfs'); jest.mock('../../util/host-rules'); hostRules = require('../../util/host-rules'); bitbucket = await import('.'); - GitStorage = require('../git').Storage; - GitStorage.mockImplementation( - () => - ({ - initRepo: jest.fn(), - cleanRepo: jest.fn(), - getFileList: jest.fn(), - branchExists: jest.fn(() => true), - isBranchStale: jest.fn(() => false), - setBaseBranch: jest.fn(), - getBranchLastCommitTime: jest.fn(), - getAllRenovateBranches: jest.fn(), - getCommitMessages: jest.fn(), - getFile: jest.fn(), - commitFiles: jest.fn(), - mergeBranch: jest.fn(), - deleteBranch: jest.fn(), - getRepoStatus: jest.fn(), - getBranchCommit: jest.fn( - () => '0d9c7726c3d628b7e28af234595cfd20febdbf8e' - ), - } as any) + gitfs = require('../../util/gitfs'); + gitfs.branchExists.mockResolvedValue(true); + gitfs.isBranchStale.mockResolvedValue(false); + gitfs.getBranchCommit.mockResolvedValue( + '0d9c7726c3d628b7e28af234595cfd20febdbf8e' ); const endpoint = scenarioName === 'endpoint with path' @@ -1807,15 +1788,7 @@ Followed by some information. }); it('throws repository-changed', async () => { - GitStorage.mockImplementationOnce( - () => - ({ - initRepo: jest.fn(), - branchExists: jest.fn(() => Promise.resolve(false)), - cleanRepo: jest.fn(), - } as any) - ); - + gitfs.branchExists.mockResolvedValue(false); await initRepo(); await expect( bitbucket.getBranchStatus('somebranch', []) diff --git a/lib/platform/bitbucket-server/index.ts b/lib/platform/bitbucket-server/index.ts index 106e20ad4e6b3e..ec8ab9d98e3a81 100644 --- a/lib/platform/bitbucket-server/index.ts +++ b/lib/platform/bitbucket-server/index.ts @@ -10,6 +10,7 @@ import { PLATFORM_TYPE_BITBUCKET_SERVER } from '../../constants/platforms'; import { PR_STATE_ALL, PR_STATE_OPEN } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; +import * as gitfs from '../../util/gitfs'; import * as hostRules from '../../util/host-rules'; import { HttpResponse } from '../../util/http'; import { @@ -34,7 +35,6 @@ import { RepoParams, VulnerabilityAlert, } from '../common'; -import GitStorage, { StatusResult } from '../git'; import { smartTruncate } from '../utils/pr-body'; import { BbbsRestPr, BbsConfig, BbsPr, BbsRestUserRef } from './types'; import * as utils from './utils'; @@ -107,9 +107,7 @@ export async function getRepos(): Promise { export function cleanRepo(): Promise { logger.debug(`cleanRepo()`); - if (config.storage) { - config.storage.cleanRepo(); - } + gitfs.cleanRepo(); config = {} as any; return Promise.resolve(); } @@ -177,7 +175,7 @@ export async function initRepo({ } const { host, pathname } = url.parse(defaults.endpoint!); - const gitUrl = GitStorage.getUrl({ + const gitUrl = gitfs.getUrl({ protocol: defaults.endpoint!.split(':')[0], auth: `${opts.username}:${opts.password}`, host: `${host}${pathname}${ @@ -186,8 +184,7 @@ export async function initRepo({ repository, }); - config.storage = new GitStorage(); - await config.storage.initRepo({ + await gitfs.initRepo({ ...config, localDir, url: gitUrl, @@ -253,21 +250,21 @@ export async function setBaseBranch( branchName: string = config.defaultBranch ): Promise { config.baseBranch = branchName; - const baseBranchSha = await config.storage.setBaseBranch(branchName); + const baseBranchSha = await gitfs.setBaseBranch(branchName); return baseBranchSha; } export /* istanbul ignore next */ function setBranchPrefix( branchPrefix: string ): Promise { - return config.storage.setBranchPrefix(branchPrefix); + return gitfs.setBranchPrefix(branchPrefix); } // Search // Get full file list export function getFileList(): Promise { - return config.storage.getFileList(); + return gitfs.getFileList(); } // Branch @@ -275,12 +272,12 @@ export function getFileList(): Promise { // Returns true if branch exists, otherwise false export function branchExists(branchName: string): Promise { logger.debug(`branchExists(${branchName})`); - return config.storage.branchExists(branchName); + return gitfs.branchExists(branchName); } export function isBranchStale(branchName: string): Promise { logger.debug(`isBranchStale(${branchName})`); - return config.storage.isBranchStale(branchName); + return gitfs.isBranchStale(branchName); } // Gets details for a PR @@ -403,7 +400,7 @@ export async function getPrList(_args?: any): Promise { /* istanbul ignore next */ export async function getPrFiles(pr: Pr): Promise { - return config.storage.getBranchFiles(pr.branchName, pr.targetBranch); + return gitfs.getBranchFiles(pr.branchName, pr.targetBranch); } // TODO: coverage @@ -442,13 +439,13 @@ export function getAllRenovateBranches( branchPrefix: string ): Promise { logger.debug('getAllRenovateBranches'); - return config.storage.getAllRenovateBranches(branchPrefix); + return gitfs.getAllRenovateBranches(branchPrefix); } export async function commitFiles( commitFilesConfig: CommitFilesConfig ): Promise { - const commit = config.storage.commitFiles(commitFilesConfig); + const commit = gitfs.commitFiles(commitFilesConfig); // wait for pr change propagation await delay(1000); @@ -459,7 +456,7 @@ export async function commitFiles( export function getFile(filePath: string, branchName: string): Promise { logger.debug(`getFile(${filePath}, ${branchName})`); - return config.storage.getFile(filePath, branchName); + return gitfs.getFile(filePath, branchName); } export async function deleteBranch( @@ -480,30 +477,30 @@ export async function deleteBranch( updatePrVersion(pr.number, body.version); } } - return config.storage.deleteBranch(branchName); + return gitfs.deleteBranch(branchName); } export function mergeBranch(branchName: string): Promise { logger.debug(`mergeBranch(${branchName})`); - return config.storage.mergeBranch(branchName); + return gitfs.mergeBranch(branchName); } export function getBranchLastCommitTime(branchName: string): Promise { logger.debug(`getBranchLastCommitTime(${branchName})`); - return config.storage.getBranchLastCommitTime(branchName); + return gitfs.getBranchLastCommitTime(branchName); } export /* istanbul ignore next */ function getRepoStatus(): Promise< - StatusResult + gitfs.StatusResult > { - return config.storage.getRepoStatus(); + return gitfs.getRepoStatus(); } async function getStatus( branchName: string, useCache = true ): Promise { - const branchCommit = await config.storage.getBranchCommit(branchName); + const branchCommit = await gitfs.getBranchCommit(branchName); return ( await bitbucketServerHttp.getJson( @@ -560,7 +557,7 @@ async function getStatusCheck( branchName: string, useCache = true ): Promise { - const branchCommit = await config.storage.getBranchCommit(branchName); + const branchCommit = await gitfs.getBranchCommit(branchName); return utils.accumulateValues( `./rest/build-status/1.0/commits/${branchCommit}`, @@ -613,7 +610,7 @@ export async function setBranchStatus({ } logger.debug({ branch: branchName, context, state }, 'Setting branch status'); - const branchCommit = await config.storage.getBranchCommit(branchName); + const branchCommit = await gitfs.getBranchCommit(branchName); try { const body: any = { @@ -1080,7 +1077,7 @@ export function getPrBody(input: string): string { export function getCommitMessages(): Promise { logger.debug(`getCommitMessages()`); - return config.storage.getCommitMessages(); + return gitfs.getCommitMessages(); } export function getVulnerabilityAlerts(): Promise { diff --git a/lib/platform/bitbucket-server/types.ts b/lib/platform/bitbucket-server/types.ts index db1c5abf0ce470..e319775cd41e0f 100644 --- a/lib/platform/bitbucket-server/types.ts +++ b/lib/platform/bitbucket-server/types.ts @@ -1,5 +1,4 @@ import { Pr } from '../common'; -import GitStorage from '../git'; export interface BbsConfig { baseBranch: string; @@ -12,7 +11,6 @@ export interface BbsConfig { projectKey: string; repository: string; repositorySlug: string; - storage: GitStorage; prVersions: Map; diff --git a/lib/platform/bitbucket/index.spec.ts b/lib/platform/bitbucket/index.spec.ts index d9e266459a12ff..cf3c01592ce001 100644 --- a/lib/platform/bitbucket/index.spec.ts +++ b/lib/platform/bitbucket/index.spec.ts @@ -3,6 +3,7 @@ import * as httpMock from '../../../test/httpMock'; import { REPOSITORY_DISABLED } from '../../constants/error-messages'; import { logger as _logger } from '../../logger'; import { BranchStatus } from '../../types'; +import * as _gitfs from '../../util/gitfs'; import { setBaseUrl } from '../../util/http/bitbucket'; import { Platform, RepoParams } from '../common'; @@ -47,37 +48,22 @@ const commits = { describe('platform/bitbucket', () => { let bitbucket: Platform; let hostRules: jest.Mocked; - let GitStorage: jest.Mocked & jest.Mock; + let gitfs: jest.Mocked; let logger: jest.Mocked; beforeEach(async () => { // reset module jest.resetModules(); httpMock.reset(); httpMock.setup(); - jest.mock('../git'); + jest.mock('../../util/gitfs'); jest.mock('../../util/host-rules'); jest.mock('../../logger'); hostRules = require('../../util/host-rules'); bitbucket = await import('.'); logger = (await import('../../logger')).logger as any; - GitStorage = require('../git').Storage; - GitStorage.mockImplementation(() => ({ - initRepo: jest.fn(), - cleanRepo: jest.fn(), - getFileList: jest.fn(), - branchExists: jest.fn(() => true), - isBranchStale: jest.fn(() => false), - setBaseBranch: jest.fn(), - getBranchLastCommitTime: jest.fn(), - getAllRenovateBranches: jest.fn(), - getCommitMessages: jest.fn(), - getFile: jest.fn(), - commitFiles: jest.fn(), - mergeBranch: jest.fn(), - deleteBranch: jest.fn(), - getRepoStatus: jest.fn(), - })); - + gitfs = require('../../util/gitfs'); + gitfs.branchExists.mockResolvedValue(true); + gitfs.isBranchStale.mockResolvedValue(false); // clean up hostRules hostRules.clear(); hostRules.find.mockReturnValue({ diff --git a/lib/platform/bitbucket/index.ts b/lib/platform/bitbucket/index.ts index 903a66b65cf754..3e1b0d07bbbf8f 100644 --- a/lib/platform/bitbucket/index.ts +++ b/lib/platform/bitbucket/index.ts @@ -10,6 +10,7 @@ import { PLATFORM_TYPE_BITBUCKET } from '../../constants/platforms'; import { PR_STATE_ALL, PR_STATE_OPEN } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; +import * as gitfs from '../../util/gitfs'; import * as hostRules from '../../util/host-rules'; import { BitbucketHttp, setBaseUrl } from '../../util/http/bitbucket'; import { sanitize } from '../../util/sanitize'; @@ -29,7 +30,6 @@ import { RepoParams, VulnerabilityAlert, } from '../common'; -import GitStorage, { StatusResult } from '../git'; import { smartTruncate } from '../utils/pr-body'; import { readOnlyIssueBody } from '../utils/read-only-issue-body'; import * as comments from './comments'; @@ -148,15 +148,14 @@ export async function initRepo({ // `api-staging.` to `staging.` const hostnameWithoutApiPrefix = /api[.|-](.+)/.exec(hostname)[1]; - const url = GitStorage.getUrl({ + const url = gitfs.getUrl({ protocol: 'https', auth: `${opts.username}:${opts.password}`, hostname: hostnameWithoutApiPrefix, repository, }); - config.storage = new GitStorage(); - await config.storage.initRepo({ + await gitfs.initRepo({ ...config, localDir, url, @@ -180,7 +179,7 @@ export function getRepoForceRebase(): Promise { // Get full file list export function getFileList(): Promise { - return config.storage.getFileList(); + return gitfs.getFileList(); } export async function setBaseBranch( @@ -189,38 +188,38 @@ export async function setBaseBranch( logger.debug(`Setting baseBranch to ${branchName}`); config.baseBranch = branchName; delete config.baseCommitSHA; - const baseBranchSha = await config.storage.setBaseBranch(branchName); + const baseBranchSha = await gitfs.setBaseBranch(branchName); return baseBranchSha; } export /* istanbul ignore next */ function setBranchPrefix( branchPrefix: string ): Promise { - return config.storage.setBranchPrefix(branchPrefix); + return gitfs.setBranchPrefix(branchPrefix); } // Branch // Returns true if branch exists, otherwise false export function branchExists(branchName: string): Promise { - return config.storage.branchExists(branchName); + return gitfs.branchExists(branchName); } export function getAllRenovateBranches( branchPrefix: string ): Promise { - return config.storage.getAllRenovateBranches(branchPrefix); + return gitfs.getAllRenovateBranches(branchPrefix); } export function isBranchStale(branchName: string): Promise { - return config.storage.isBranchStale(branchName); + return gitfs.isBranchStale(branchName); } export function getFile( filePath: string, branchName?: string ): Promise { - return config.storage.getFile(filePath, branchName); + return gitfs.getFile(filePath, branchName); } // istanbul ignore next @@ -249,7 +248,7 @@ export async function getPrList(): Promise { /* istanbul ignore next */ export async function getPrFiles(pr: Pr): Promise { - return config.storage.getBranchFiles(pr.branchName, pr.targetBranch); + return gitfs.getBranchFiles(pr.branchName, pr.targetBranch); } export async function findPr({ @@ -283,31 +282,31 @@ export async function deleteBranch( ); } } - return config.storage.deleteBranch(branchName); + return gitfs.deleteBranch(branchName); } export function getBranchLastCommitTime(branchName: string): Promise { - return config.storage.getBranchLastCommitTime(branchName); + return gitfs.getBranchLastCommitTime(branchName); } // istanbul ignore next -export function getRepoStatus(): Promise { - return config.storage.getRepoStatus(); +export function getRepoStatus(): Promise { + return gitfs.getRepoStatus(); } export function mergeBranch(branchName: string): Promise { - return config.storage.mergeBranch(branchName); + return gitfs.mergeBranch(branchName); } // istanbul ignore next export function commitFiles( commitFilesConfig: CommitFilesConfig ): Promise { - return config.storage.commitFiles(commitFilesConfig); + return gitfs.commitFiles(commitFilesConfig); } export function getCommitMessages(): Promise { - return config.storage.getCommitMessages(); + return gitfs.getCommitMessages(); } async function isPrConflicted(prNo: number): Promise { @@ -874,10 +873,7 @@ export async function mergePr( // Pull Request export function cleanRepo(): Promise { - // istanbul ignore if - if (config.storage && config.storage.cleanRepo) { - config.storage.cleanRepo(); - } + gitfs.cleanRepo(); config = {} as any; return Promise.resolve(); } diff --git a/lib/platform/bitbucket/utils.ts b/lib/platform/bitbucket/utils.ts index 0c4e6b4a5d6b9f..c52f13a29e9f33 100644 --- a/lib/platform/bitbucket/utils.ts +++ b/lib/platform/bitbucket/utils.ts @@ -4,7 +4,6 @@ import { BranchStatus } from '../../types'; import { HttpResponse } from '../../util/http'; import { BitbucketHttp } from '../../util/http/bitbucket'; import { Pr } from '../common'; -import { Storage } from '../git'; const bitbucketHttp = new BitbucketHttp(); @@ -17,7 +16,6 @@ export interface Config { owner: string; prList: Pr[]; repository: string; - storage: Storage; bbUseDefaultReviewers: boolean; username: string; diff --git a/lib/platform/git/index.ts b/lib/platform/git/index.ts deleted file mode 100644 index 0cc66884affc17..00000000000000 --- a/lib/platform/git/index.ts +++ /dev/null @@ -1,600 +0,0 @@ -import { join } from 'path'; -import URL from 'url'; -import fs from 'fs-extra'; -import Git from 'simple-git/promise'; -import { - CONFIG_VALIDATION, - REPOSITORY_CHANGED, - REPOSITORY_EMPTY, - REPOSITORY_TEMPORARY_ERROR, - SYSTEM_INSUFFICIENT_DISK_SPACE, -} from '../../constants/error-messages'; -import { logger } from '../../logger'; -import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as limits from '../../workers/global/limits'; -import { CommitFilesConfig } from '../common'; -import { writePrivateKey } from './private-key'; - -declare module 'fs-extra' { - export function exists(pathLike: string): Promise; -} - -export type StatusResult = Git.StatusResult; - -export type DiffResult = Git.DiffResult; - -interface StorageConfig { - localDir: string; - baseBranch?: string; - url: string; - extraCloneOpts?: Git.Options; - gitAuthorName?: string; - gitAuthorEmail?: string; -} - -interface LocalConfig extends StorageConfig { - baseBranch: string; - baseBranchSha: string; - branchExists: Record; - branchPrefix: string; -} - -// istanbul ignore next -function checkForPlatformFailure(err: Error): void { - if (process.env.NODE_ENV === 'test') { - return; - } - const platformFailureStrings = [ - 'remote: Invalid username or password', - 'gnutls_handshake() failed', - 'The requested URL returned error: 5', - 'The remote end hung up unexpectedly', - 'access denied or repository not exported', - 'Could not write new index file', - 'Failed to connect to', - 'Connection timed out', - ]; - for (const errorStr of platformFailureStrings) { - if (err.message.includes(errorStr)) { - throw new ExternalHostError(err, 'git'); - } - } -} - -function localName(branchName: string): string { - return branchName.replace(/^origin\//, ''); -} - -function throwBaseBranchValidationError(branchName: string): never { - const error = new Error(CONFIG_VALIDATION); - error.validationError = 'baseBranch not found'; - error.validationMessage = - 'The following configured baseBranch could not be found: ' + branchName; - throw error; -} - -async function isDirectory(dir: string): Promise { - try { - return (await fs.stat(dir)).isDirectory(); - } catch (err) { - return false; - } -} - -export class Storage { - private _config: LocalConfig = {} as any; - - private _git: Git.SimpleGit | undefined; - - private _cwd: string | undefined; - - private _privateKeySet = false; - - private async _resetToBranch(branchName: string): Promise { - logger.debug(`resetToBranch(${branchName})`); - await this._git.raw(['reset', '--hard']); - await this._git.checkout(branchName); - await this._git.raw(['reset', '--hard', 'origin/' + branchName]); - await this._git.raw(['clean', '-fd']); - } - - private async _cleanLocalBranches(): Promise { - const existingBranches = (await this._git.raw(['branch'])) - .split('\n') - .map((branch) => branch.trim()) - .filter((branch) => branch.length) - .filter((branch) => !branch.startsWith('* ')); - logger.debug({ existingBranches }); - for (const branchName of existingBranches) { - await this._deleteLocalBranch(branchName); - } - } - - async initRepo(args: StorageConfig): Promise { - this.cleanRepo(); - // eslint-disable-next-line no-multi-assign - const config: LocalConfig = (this._config = { ...args } as any); - // eslint-disable-next-line no-multi-assign - const cwd = (this._cwd = config.localDir); - this._config.branchExists = {}; - logger.debug('Initializing git repository into ' + cwd); - const gitHead = join(cwd, '.git/HEAD'); - let clone = true; - - // TODO: move to private class scope - async function setBaseBranchToDefault(git: Git.SimpleGit): Promise { - // see https://stackoverflow.com/a/44750379/1438522 - try { - config.baseBranch = - config.baseBranch || - (await git.raw(['symbolic-ref', 'refs/remotes/origin/HEAD'])) - .replace('refs/remotes/origin/', '') - .trim(); - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - if ( - err.message.startsWith( - 'fatal: ref refs/remotes/origin/HEAD is not a symbolic ref' - ) - ) { - throw new Error(REPOSITORY_EMPTY); - } - throw err; - } - } - - if (await fs.exists(gitHead)) { - try { - this._git = Git(cwd).silent(true); - await this._git.raw(['remote', 'set-url', 'origin', config.url]); - const fetchStart = Date.now(); - await this._git.fetch(['--depth=10']); - await setBaseBranchToDefault(this._git); - await this._resetToBranch(config.baseBranch); - await this._cleanLocalBranches(); - await this._git.raw(['remote', 'prune', 'origin']); - const durationMs = Math.round(Date.now() - fetchStart); - logger.debug({ durationMs }, 'git fetch completed'); - clone = false; - } catch (err) /* istanbul ignore next */ { - logger.error({ err }, 'git fetch error'); - } - } - if (clone) { - await fs.emptyDir(cwd); - this._git = Git(cwd).silent(true); - const cloneStart = Date.now(); - try { - // clone only the default branch - let opts = ['--depth=2']; - if (config.extraCloneOpts) { - opts = opts.concat( - Object.entries(config.extraCloneOpts).map((e) => `${e[0]}=${e[1]}`) - ); - } - await this._git.clone(config.url, '.', opts); - } catch (err) /* istanbul ignore next */ { - logger.debug({ err }, 'git clone error'); - if (err.message?.includes('write error: No space left on device')) { - throw new Error(SYSTEM_INSUFFICIENT_DISK_SPACE); - } - throw new ExternalHostError(err, 'git'); - } - const durationMs = Math.round(Date.now() - cloneStart); - logger.debug({ durationMs }, 'git clone completed'); - } - const submodules = await this.getSubmodules(); - for (const submodule of submodules) { - try { - logger.debug(`Cloning git submodule at ${submodule}`); - await this._git.submoduleUpdate(['--init', '--', submodule]); - } catch (err) { - logger.warn(`Unable to initialise git submodule at ${submodule}`); - } - } - try { - const latestCommitDate = (await this._git.log({ n: 1 })).latest.date; - logger.debug({ latestCommitDate }, 'latest commit'); - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - if (err.message.includes('does not have any commits yet')) { - throw new Error(REPOSITORY_EMPTY); - } - logger.warn({ err }, 'Cannot retrieve latest commit date'); - } - try { - const { gitAuthorName, gitAuthorEmail } = args; - if (gitAuthorName) { - logger.debug({ gitAuthorName }, 'Setting git author name'); - await this._git.raw(['config', 'user.name', gitAuthorName]); - } - if (gitAuthorEmail) { - logger.debug({ gitAuthorEmail }, 'Setting git author email'); - await this._git.raw(['config', 'user.email', gitAuthorEmail]); - } - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - logger.debug({ err }, 'Error setting git author config'); - throw new Error(REPOSITORY_TEMPORARY_ERROR); - } - - await setBaseBranchToDefault(this._git); - } - - // istanbul ignore next - getRepoStatus(): Promise { - return this._git.status(); - } - - async createBranch(branchName: string, sha: string): Promise { - logger.debug(`createBranch(${branchName})`); - await this._git.reset('hard'); - await this._git.raw(['clean', '-fd']); - await this._git.checkout(['-B', branchName, sha]); - await this._git.push('origin', branchName, { '--force': true }); - this._config.branchExists[branchName] = true; - } - - // Return the commit SHA for a branch - async getBranchCommit(branchName: string): Promise { - if (!(await this.branchExists(branchName))) { - throw Error( - 'Cannot fetch commit for branch that does not exist: ' + branchName - ); - } - const res = await this._git.revparse(['origin/' + branchName]); - return res.trim(); - } - - async getCommitMessages(): Promise { - logger.debug('getCommitMessages'); - const res = await this._git.log({ - n: 10, - format: { message: '%s' }, - }); - return res.all.map((commit) => commit.message); - } - - async setBaseBranch(branchName: string): Promise { - if (branchName) { - if (!(await this.branchExists(branchName))) { - throwBaseBranchValidationError(branchName); - } - logger.debug(`Setting baseBranch to ${branchName}`); - this._config.baseBranch = branchName; - try { - if (branchName !== 'master') { - this._config.baseBranchSha = ( - await this._git.raw(['rev-parse', 'origin/' + branchName]) - ).trim(); - } - await this._git.checkout([branchName, '-f']); - await this._git.reset('hard'); - const latestCommitDate = (await this._git.log({ n: 1 })).latest.date; - logger.debug({ branchName, latestCommitDate }, 'latest commit'); - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - if ( - err.message.includes( - 'unknown revision or path not in the working tree' - ) || - err.message.includes('did not match any file(s) known to git') - ) { - throwBaseBranchValidationError(branchName); - } - throw err; - } - } - return ( - this._config.baseBranchSha || - (await this._git.raw(['rev-parse', 'origin/master'])).trim() - ); - } - - /* - * When we initially clone, we clone only the default branch so how no knowledge of other branches existing. - * By calling this function once the repo's branchPrefix is known, we can fetch all of Renovate's branches in one command. - */ - async setBranchPrefix(branchPrefix: string): Promise { - logger.debug('Setting branchPrefix: ' + branchPrefix); - this._config.branchPrefix = branchPrefix; - const ref = `refs/heads/${branchPrefix}*:refs/remotes/origin/${branchPrefix}*`; - try { - await this._git.fetch(['origin', ref, '--depth=2', '--force']); - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - throw err; - } - } - - async getFileList(): Promise { - const branch = this._config.baseBranch; - const submodules = await this.getSubmodules(); - const files: string = await this._git.raw(['ls-tree', '-r', branch]); - // istanbul ignore if - if (!files) { - return []; - } - return files - .split('\n') - .filter(Boolean) - .filter((line) => line.startsWith('100')) - .map((line) => line.split(/\t/).pop()) - .filter((file: string) => - submodules.every((submodule: string) => !file.startsWith(submodule)) - ); - } - - async getSubmodules(): Promise { - return ( - (await this._git.raw([ - 'config', - '--file', - '.gitmodules', - '--get-regexp', - 'path', - ])) || '' - ) - .trim() - .split(/[\n\s]/) - .filter((_e: string, i: number) => i % 2); - } - - async branchExists(branchName: string): Promise { - // First check cache - if (this._config.branchExists[branchName] !== undefined) { - return this._config.branchExists[branchName]; - } - if (!branchName.startsWith(this._config.branchPrefix)) { - // fetch the branch only if it's not part of the existing branchPrefix - try { - await this._git.raw([ - 'remote', - 'set-branches', - '--add', - 'origin', - branchName, - ]); - await this._git.fetch(['origin', branchName, '--depth=2']); - } catch (err) { - checkForPlatformFailure(err); - } - } - try { - await this._git.raw(['show-branch', 'origin/' + branchName]); - this._config.branchExists[branchName] = true; - return true; - } catch (err) { - checkForPlatformFailure(err); - this._config.branchExists[branchName] = false; - return false; - } - } - - async getAllRenovateBranches(branchPrefix: string): Promise { - const branches = await this._git.branch(['--remotes', '--verbose']); - return branches.all - .map(localName) - .filter((branchName) => branchName.startsWith(branchPrefix)); - } - - async isBranchStale(branchName: string): Promise { - if (!(await this.branchExists(branchName))) { - throw Error( - 'Cannot check staleness for branch that does not exist: ' + branchName - ); - } - const branches = await this._git.branch([ - '--remotes', - '--verbose', - '--contains', - this._config.baseBranchSha || `origin/${this._config.baseBranch}`, - ]); - return !branches.all.map(localName).includes(branchName); - } - - private async _deleteLocalBranch(branchName: string): Promise { - await this._git.branch(['-D', branchName]); - } - - async deleteBranch(branchName: string): Promise { - try { - await this._git.raw(['push', '--delete', 'origin', branchName]); - logger.debug({ branchName }, 'Deleted remote branch'); - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - logger.debug({ branchName }, 'No remote branch to delete'); - } - try { - await this._deleteLocalBranch(branchName); - // istanbul ignore next - logger.debug({ branchName }, 'Deleted local branch'); - } catch (err) { - checkForPlatformFailure(err); - logger.debug({ branchName }, 'No local branch to delete'); - } - this._config.branchExists[branchName] = false; - } - - async mergeBranch(branchName: string): Promise { - await this._git.reset('hard'); - await this._git.checkout(['-B', branchName, 'origin/' + branchName]); - await this._git.checkout(this._config.baseBranch); - await this._git.merge(['--ff-only', branchName]); - await this._git.push('origin', this._config.baseBranch); - limits.incrementLimit('prCommitsPerRunLimit'); - } - - async getBranchLastCommitTime(branchName: string): Promise { - try { - const time = await this._git.show([ - '-s', - '--format=%ai', - 'origin/' + branchName, - ]); - return new Date(Date.parse(time)); - } catch (err) { - checkForPlatformFailure(err); - return new Date(); - } - } - - async getBranchFiles( - branchName: string, - baseBranchName?: string - ): Promise { - try { - const diff = await this._git.diffSummary([ - branchName, - baseBranchName || this._config.baseBranch, - ]); - return diff.files.map((file) => file.file); - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - return null; - } - } - - async getFile(filePath: string, branchName?: string): Promise { - if (branchName) { - const exists = await this.branchExists(branchName); - if (!exists) { - logger.debug({ branchName }, 'branch no longer exists - aborting'); - throw new Error(REPOSITORY_CHANGED); - } - } - try { - const content = await this._git.show([ - 'origin/' + (branchName || this._config.baseBranch) + ':' + filePath, - ]); - return content; - } catch (err) { - checkForPlatformFailure(err); - return null; - } - } - - async hasDiff(branchName: string): Promise { - try { - return (await this._git.diff(['HEAD', branchName])) !== ''; - } catch (err) { - return true; - } - } - - async commitFiles({ - branchName, - files, - message, - force = false, - }: CommitFilesConfig): Promise { - logger.debug(`Committing files to branch ${branchName}`); - if (!this._privateKeySet) { - await writePrivateKey(this._cwd); - this._privateKeySet = true; - } - try { - await this._git.reset('hard'); - await this._git.raw(['clean', '-fd']); - await this._git.checkout([ - '-B', - branchName, - 'origin/' + this._config.baseBranch, - ]); - const fileNames = []; - const deleted = []; - for (const file of files) { - // istanbul ignore if - if (file.name === '|delete|') { - deleted.push(file.contents); - } else if (await isDirectory(join(this._cwd, file.name))) { - fileNames.push(file.name); - await this._git.add(file.name); - } else { - fileNames.push(file.name); - let contents; - // istanbul ignore else - if (typeof file.contents === 'string') { - contents = Buffer.from(file.contents); - } else { - contents = file.contents; - } - await fs.outputFile(join(this._cwd, file.name), contents); - } - } - // istanbul ignore if - if (fileNames.length === 1 && fileNames[0] === 'renovate.json') { - fileNames.unshift('-f'); - } - if (fileNames.length) { - await this._git.add(fileNames); - } - if (deleted.length) { - for (const f of deleted) { - try { - await this._git.rm([f]); - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - logger.debug({ err }, 'Cannot delete ' + f); - } - } - } - const commitRes = await this._git.commit(message, [], { - '--no-verify': true, - }); - const commit = commitRes?.commit || 'unknown'; - if (!force && !(await this.hasDiff(`origin/${branchName}`))) { - logger.debug( - { branchName, fileNames }, - 'No file changes detected. Skipping commit' - ); - return null; - } - await this._git.push('origin', `${branchName}:${branchName}`, { - '--force': true, - '-u': true, - '--no-verify': true, - }); - // Fetch it after create - const ref = `refs/heads/${branchName}:refs/remotes/origin/${branchName}`; - await this._git.fetch(['origin', ref, '--depth=2', '--force']); - this._config.branchExists[branchName] = true; - limits.incrementLimit('prCommitsPerRunLimit'); - return commit; - } catch (err) /* istanbul ignore next */ { - checkForPlatformFailure(err); - logger.debug({ err }, 'Error commiting files'); - throw new Error(REPOSITORY_CHANGED); - } - } - - // eslint-disable-next-line - cleanRepo(): void {} - - static getUrl({ - protocol, - auth, - hostname, - host, - repository, - }: { - protocol?: 'ssh' | 'http' | 'https'; - auth?: string; - hostname?: string; - host?: string; - repository: string; - }): string { - if (protocol === 'ssh') { - return `git@${hostname}:${repository}.git`; - } - return URL.format({ - protocol: protocol || 'https', - auth, - hostname, - host, - pathname: repository + '.git', - }); - } -} - -export default Storage; diff --git a/lib/platform/gitea/index.spec.ts b/lib/platform/gitea/index.spec.ts index 3479e56952e268..d95ae4aedfc646 100644 --- a/lib/platform/gitea/index.spec.ts +++ b/lib/platform/gitea/index.spec.ts @@ -18,6 +18,7 @@ import { } from '../../constants/error-messages'; import { logger as _logger } from '../../logger'; import { BranchStatus } from '../../types'; +import * as _gitfs from '../../util/gitfs'; import { setBaseUrl } from '../../util/http/gitea'; import * as ght from './gitea-helper'; @@ -25,7 +26,7 @@ describe('platform/gitea', () => { let gitea: Platform; let helper: jest.Mocked; let logger: jest.Mocked; - let GitStorage: jest.Mocked & jest.Mock; + let gitfs: jest.Mocked; const mockCommitHash = '0d9c7726c3d628b7e28af234595cfd20febdbf8e'; @@ -152,53 +153,19 @@ describe('platform/gitea', () => { }, ]; - const gsmInitRepo = jest.fn(); - const gsmCleanRepo = jest.fn(); - const gsmSetBaseBranch = jest.fn(); - const gsmGetCommitMessages = jest.fn(); - const gsmGetAllRenovateBranches = jest.fn(); - const gsmGetFileList = jest.fn(); - const gsmGetRepoStatus = jest.fn(); - const gsmGetFile = jest.fn(); - const gsmGetBranchLastCommitTime = jest.fn(); - const gsmMergeBranch = jest.fn(); - const gsmBranchExists = jest.fn(); - const gsmSetBranchPrefix = jest.fn(); - const gsmCommitFilesToBranch = jest.fn(); - const gsmDeleteBranch = jest.fn(); - const gsmIsBranchStale = jest.fn(() => false); - const gsmGetBranchCommit = jest.fn(() => mockCommitHash); - beforeEach(async () => { jest.resetModules(); jest.clearAllMocks(); jest.mock('./gitea-helper'); - jest.mock('../git'); + jest.mock('../../util/gitfs'); jest.mock('../../logger'); gitea = await import('.'); helper = (await import('./gitea-helper')) as any; logger = (await import('../../logger')).logger as any; - GitStorage = (await import('../git')).Storage as any; - - GitStorage.mockImplementation(() => ({ - initRepo: gsmInitRepo, - cleanRepo: gsmCleanRepo, - setBaseBranch: gsmSetBaseBranch, - getCommitMessages: gsmGetCommitMessages, - getAllRenovateBranches: gsmGetAllRenovateBranches, - getFileList: gsmGetFileList, - getRepoStatus: gsmGetRepoStatus, - getFile: gsmGetFile, - getBranchLastCommitTime: gsmGetBranchLastCommitTime, - mergeBranch: gsmMergeBranch, - branchExists: gsmBranchExists, - setBranchPrefix: gsmSetBranchPrefix, - isBranchStale: gsmIsBranchStale, - getBranchCommit: gsmGetBranchCommit, - commitFiles: gsmCommitFilesToBranch, - deleteBranch: gsmDeleteBranch, - })); + gitfs = require('../../util/gitfs'); + gitfs.isBranchStale.mockResolvedValue(false); + gitfs.getBranchCommit.mockResolvedValue(mockCommitHash); global.gitAuthor = { name: 'Renovate', email: 'renovate@example.com' }; @@ -364,13 +331,13 @@ describe('platform/gitea', () => { describe('cleanRepo', () => { it('does not throw an error with uninitialized repo', async () => { await gitea.cleanRepo(); - expect(gsmCleanRepo).not.toHaveBeenCalled(); + expect(gitfs.cleanRepo).toHaveBeenCalledTimes(1); }); it('propagates call to storage class with initialized repo', async () => { await initFakeRepo(); await gitea.cleanRepo(); - expect(gsmCleanRepo).toHaveBeenCalledTimes(1); + expect(gitfs.cleanRepo).toHaveBeenCalledTimes(1); }); }); @@ -443,16 +410,16 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.setBaseBranch(); - expect(gsmSetBaseBranch).toHaveBeenCalledTimes(1); - expect(gsmSetBaseBranch).toHaveBeenCalledWith(mockRepo.default_branch); + expect(gitfs.setBaseBranch).toHaveBeenCalledTimes(1); + expect(gitfs.setBaseBranch).toHaveBeenCalledWith(mockRepo.default_branch); }); it('should set custom base branch', async () => { await initFakeRepo(); await gitea.setBaseBranch('devel'); - expect(gsmSetBaseBranch).toHaveBeenCalledTimes(1); - expect(gsmSetBaseBranch).toHaveBeenCalledWith('devel'); + expect(gitfs.setBaseBranch).toHaveBeenCalledTimes(1); + expect(gitfs.setBaseBranch).toHaveBeenCalledWith('devel'); }); }); @@ -1322,8 +1289,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.deleteBranch('some-branch'); - expect(gsmDeleteBranch).toHaveBeenCalledTimes(1); - expect(gsmDeleteBranch).toHaveBeenCalledWith('some-branch'); + expect(gitfs.deleteBranch).toHaveBeenCalledTimes(1); + expect(gitfs.deleteBranch).toHaveBeenCalledWith('some-branch'); }); it('should not close pull request by default', async () => { @@ -1344,8 +1311,8 @@ describe('platform/gitea', () => { mockRepo.full_name, mockPR.number ); - expect(gsmDeleteBranch).toHaveBeenCalledTimes(1); - expect(gsmDeleteBranch).toHaveBeenCalledWith(mockPR.head.label); + expect(gitfs.deleteBranch).toHaveBeenCalledTimes(1); + expect(gitfs.deleteBranch).toHaveBeenCalledWith(mockPR.head.label); }); it('should skip closing pull request if missing', async () => { @@ -1354,8 +1321,8 @@ describe('platform/gitea', () => { await gitea.deleteBranch('missing', true); expect(helper.closePR).not.toHaveBeenCalled(); - expect(gsmDeleteBranch).toHaveBeenCalledTimes(1); - expect(gsmDeleteBranch).toHaveBeenCalledWith('missing'); + expect(gitfs.deleteBranch).toHaveBeenCalledTimes(1); + expect(gitfs.deleteBranch).toHaveBeenCalledWith('missing'); }); }); @@ -1391,8 +1358,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.commitFiles(commitConfig); - expect(gsmCommitFilesToBranch).toHaveBeenCalledTimes(1); - expect(gsmCommitFilesToBranch).toHaveBeenCalledWith({ + expect(gitfs.commitFiles).toHaveBeenCalledTimes(1); + expect(gitfs.commitFiles).toHaveBeenCalledWith({ ...commitConfig, }); }); @@ -1411,8 +1378,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.isBranchStale('some-branch'); - expect(gsmIsBranchStale).toHaveBeenCalledTimes(1); - expect(gsmIsBranchStale).toHaveBeenCalledWith('some-branch'); + expect(gitfs.isBranchStale).toHaveBeenCalledTimes(1); + expect(gitfs.isBranchStale).toHaveBeenCalledWith('some-branch'); }); }); @@ -1421,8 +1388,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.setBranchPrefix('some-branch'); - expect(gsmSetBranchPrefix).toHaveBeenCalledTimes(1); - expect(gsmSetBranchPrefix).toHaveBeenCalledWith('some-branch'); + expect(gitfs.setBranchPrefix).toHaveBeenCalledTimes(1); + expect(gitfs.setBranchPrefix).toHaveBeenCalledWith('some-branch'); }); }); @@ -1431,8 +1398,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.branchExists('some-branch'); - expect(gsmBranchExists).toHaveBeenCalledTimes(1); - expect(gsmBranchExists).toHaveBeenCalledWith('some-branch'); + expect(gitfs.branchExists).toHaveBeenCalledTimes(1); + expect(gitfs.branchExists).toHaveBeenCalledWith('some-branch'); }); }); @@ -1441,8 +1408,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.mergeBranch('some-branch'); - expect(gsmMergeBranch).toHaveBeenCalledTimes(1); - expect(gsmMergeBranch).toHaveBeenCalledWith('some-branch'); + expect(gitfs.mergeBranch).toHaveBeenCalledTimes(1); + expect(gitfs.mergeBranch).toHaveBeenCalledWith('some-branch'); }); }); @@ -1451,8 +1418,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.getBranchLastCommitTime('some-branch'); - expect(gsmGetBranchLastCommitTime).toHaveBeenCalledTimes(1); - expect(gsmGetBranchLastCommitTime).toHaveBeenCalledWith('some-branch'); + expect(gitfs.getBranchLastCommitTime).toHaveBeenCalledTimes(1); + expect(gitfs.getBranchLastCommitTime).toHaveBeenCalledWith('some-branch'); }); }); @@ -1461,8 +1428,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.getFile('some-file', 'some-branch'); - expect(gsmGetFile).toHaveBeenCalledTimes(1); - expect(gsmGetFile).toHaveBeenCalledWith('some-file', 'some-branch'); + expect(gitfs.getFile).toHaveBeenCalledTimes(1); + expect(gitfs.getFile).toHaveBeenCalledWith('some-file', 'some-branch'); }); }); @@ -1471,7 +1438,7 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.getRepoStatus(); - expect(gsmGetRepoStatus).toHaveBeenCalledTimes(1); + expect(gitfs.getRepoStatus).toHaveBeenCalledTimes(1); }); }); @@ -1480,7 +1447,7 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.getFileList(); - expect(gsmGetFileList).toHaveBeenCalledTimes(1); + expect(gitfs.getFileList).toHaveBeenCalledTimes(1); }); }); @@ -1489,8 +1456,8 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.getAllRenovateBranches('some-prefix'); - expect(gsmGetAllRenovateBranches).toHaveBeenCalledTimes(1); - expect(gsmGetAllRenovateBranches).toHaveBeenCalledWith('some-prefix'); + expect(gitfs.getAllRenovateBranches).toHaveBeenCalledTimes(1); + expect(gitfs.getAllRenovateBranches).toHaveBeenCalledWith('some-prefix'); }); }); @@ -1499,7 +1466,7 @@ describe('platform/gitea', () => { await initFakeRepo(); await gitea.getCommitMessages(); - expect(gsmGetCommitMessages).toHaveBeenCalledTimes(1); + expect(gitfs.getCommitMessages).toHaveBeenCalledTimes(1); }); }); diff --git a/lib/platform/gitea/index.ts b/lib/platform/gitea/index.ts index 9bfe2e72e592b5..c96e8ca5c4dcad 100644 --- a/lib/platform/gitea/index.ts +++ b/lib/platform/gitea/index.ts @@ -14,6 +14,7 @@ import { PLATFORM_TYPE_GITEA } from '../../constants/platforms'; import { PR_STATE_ALL, PR_STATE_OPEN } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; +import * as gitfs from '../../util/gitfs'; import * as hostRules from '../../util/host-rules'; import { setBaseUrl } from '../../util/http/gitea'; import { sanitize } from '../../util/sanitize'; @@ -34,7 +35,6 @@ import { RepoParams, VulnerabilityAlert, } from '../common'; -import GitStorage, { StatusResult } from '../git'; import { smartTruncate } from '../utils/pr-body'; import * as helper from './gitea-helper'; @@ -44,7 +44,6 @@ type GiteaRenovateConfig = { } & RenovateConfig; interface GiteaRepoConfig { - storage: GitStorage; repository: string; localDir: string; defaultBranch: string; @@ -334,8 +333,7 @@ const platform: Platform = { gitEndpoint.auth = opts.token; // Initialize Git storage - config.storage = new GitStorage(); - await config.storage.initRepo({ + await gitfs.initRepo({ ...config, url: URL.format(gitEndpoint), gitAuthorName: global.gitAuthor?.name, @@ -365,9 +363,7 @@ const platform: Platform = { }, cleanRepo(): Promise { - if (config.storage) { - config.storage.cleanRepo(); - } + gitfs.cleanRepo(); config = {} as any; return Promise.resolve(); }, @@ -381,7 +377,7 @@ const platform: Platform = { }: BranchStatusConfig): Promise { try { // Create new status for branch commit - const branchCommit = await config.storage.getBranchCommit(branchName); + const branchCommit = await gitfs.getBranchCommit(branchName); await helper.createCommitStatus(config.repository, branchCommit, { state: helper.renovateToGiteaStatusMapping[state] || 'pending', context, @@ -460,7 +456,7 @@ const platform: Platform = { baseBranch: string = config.defaultBranch ): Promise { config.baseBranch = baseBranch; - const baseBranchSha = await config.storage.setBaseBranch(baseBranch); + const baseBranchSha = await gitfs.setBaseBranch(baseBranch); return baseBranchSha; }, @@ -480,7 +476,7 @@ const platform: Platform = { /* istanbul ignore next */ async getPrFiles(pr: Pr): Promise { - return config.storage.getBranchFiles(pr.branchName, pr.targetBranch); + return gitfs.getBranchFiles(pr.branchName, pr.targetBranch); }, async getPr(number: number): Promise { @@ -842,7 +838,7 @@ const platform: Platform = { } } - return config.storage.deleteBranch(branchName); + return gitfs.deleteBranch(branchName); }, async addAssignees(number: number, assignees: string[]): Promise { @@ -861,7 +857,7 @@ const platform: Platform = { }, commitFiles(commitFilesConfig: CommitFilesConfig): Promise { - return config.storage.commitFiles(commitFilesConfig); + return gitfs.commitFiles(commitFilesConfig); }, getPrBody(prBody: string): string { @@ -869,43 +865,43 @@ const platform: Platform = { }, isBranchStale(branchName: string): Promise { - return config.storage.isBranchStale(branchName); + return gitfs.isBranchStale(branchName); }, setBranchPrefix(branchPrefix: string): Promise { - return config.storage.setBranchPrefix(branchPrefix); + return gitfs.setBranchPrefix(branchPrefix); }, branchExists(branchName: string): Promise { - return config.storage.branchExists(branchName); + return gitfs.branchExists(branchName); }, mergeBranch(branchName: string): Promise { - return config.storage.mergeBranch(branchName); + return gitfs.mergeBranch(branchName); }, getBranchLastCommitTime(branchName: string): Promise { - return config.storage.getBranchLastCommitTime(branchName); + return gitfs.getBranchLastCommitTime(branchName); }, getFile(lockFileName: string, branchName?: string): Promise { - return config.storage.getFile(lockFileName, branchName); + return gitfs.getFile(lockFileName, branchName); }, - getRepoStatus(): Promise { - return config.storage.getRepoStatus(); + getRepoStatus(): Promise { + return gitfs.getRepoStatus(); }, getFileList(): Promise { - return config.storage.getFileList(); + return gitfs.getFileList(); }, getAllRenovateBranches(branchPrefix: string): Promise { - return config.storage.getAllRenovateBranches(branchPrefix); + return gitfs.getAllRenovateBranches(branchPrefix); }, getCommitMessages(): Promise { - return config.storage.getCommitMessages(); + return gitfs.getCommitMessages(); }, getVulnerabilityAlerts(): Promise { diff --git a/lib/platform/github/index.spec.ts b/lib/platform/github/index.spec.ts index ce5cf336c7b87e..a287c9ee0d570d 100644 --- a/lib/platform/github/index.spec.ts +++ b/lib/platform/github/index.spec.ts @@ -7,6 +7,7 @@ import { REPOSITORY_RENAMED, } from '../../constants/error-messages'; import { BranchStatus } from '../../types'; +import * as _gitfs from '../../util/gitfs'; import { Platform } from '../common'; const githubApiHost = 'https://api.github.com'; @@ -14,7 +15,7 @@ const githubApiHost = 'https://api.github.com'; describe('platform/github', () => { let github: Platform; let hostRules: jest.Mocked; - let GitStorage: jest.Mock; + let gitfs: jest.Mocked; beforeEach(async () => { // reset module jest.resetModules(); @@ -23,29 +24,12 @@ describe('platform/github', () => { jest.mock('../../util/host-rules'); github = await import('.'); hostRules = mocked(await import('../../util/host-rules')); - jest.mock('../git'); - GitStorage = (await import('../git')).Storage as any; - GitStorage.mockImplementation( - () => - ({ - initRepo: jest.fn(), - cleanRepo: jest.fn(), - getFileList: jest.fn(), - branchExists: jest.fn(() => true), - isBranchStale: jest.fn(() => false), - setBaseBranch: jest.fn(), - getBranchLastCommitTime: jest.fn(), - getAllRenovateBranches: jest.fn(), - getCommitMessages: jest.fn(), - getFile: jest.fn(), - commitFiles: jest.fn(), - mergeBranch: jest.fn(), - deleteBranch: jest.fn(), - getRepoStatus: jest.fn(), - getBranchCommit: jest.fn( - () => '0d9c7726c3d628b7e28af234595cfd20febdbf8e' - ), - } as any) + jest.mock('../../util/gitfs'); + gitfs = mocked(await import('../../util/gitfs')); + gitfs.branchExists.mockResolvedValue(true); + gitfs.isBranchStale.mockResolvedValue(true); + gitfs.getBranchCommit.mockResolvedValue( + '0d9c7726c3d628b7e28af234595cfd20febdbf8e' ); delete global.gitAuthor; hostRules.find.mockReturnValue({ diff --git a/lib/platform/github/index.ts b/lib/platform/github/index.ts index 3d6381109e32ac..1e0403ee822258 100644 --- a/lib/platform/github/index.ts +++ b/lib/platform/github/index.ts @@ -24,6 +24,7 @@ import { import { logger } from '../../logger'; import { BranchStatus } from '../../types'; import { ExternalHostError } from '../../types/errors/external-host-error'; +import * as gitfs from '../../util/gitfs'; import * as hostRules from '../../util/host-rules'; import * as githubHttp from '../../util/http/github'; import { sanitize } from '../../util/sanitize'; @@ -44,7 +45,6 @@ import { RepoParams, VulnerabilityAlert, } from '../common'; -import GitStorage, { StatusResult } from '../git'; import { smartTruncate } from '../utils/pr-body'; import { BranchProtection, @@ -150,10 +150,7 @@ export async function getRepos(): Promise { } export function cleanRepo(): Promise { - // istanbul ignore if - if (config.storage) { - config.storage.cleanRepo(); - } + gitfs.cleanRepo(); // In theory most of this isn't necessary. In practice.. config = {} as any; return Promise.resolve(); @@ -432,8 +429,7 @@ export async function initRepo({ ); parsedEndpoint.pathname = config.repository + '.git'; const url = URL.format(parsedEndpoint); - config.storage = new GitStorage(); - await config.storage.initRepo({ + await gitfs.initRepo({ ...config, url, gitAuthorName: global.gitAuthor?.name, @@ -497,39 +493,39 @@ export async function setBaseBranch( ): Promise { config.baseBranch = branchName; config.baseCommitSHA = null; - const baseBranchSha = await config.storage.setBaseBranch(branchName); + const baseBranchSha = await gitfs.setBaseBranch(branchName); return baseBranchSha; } // istanbul ignore next export function setBranchPrefix(branchPrefix: string): Promise { - return config.storage.setBranchPrefix(branchPrefix); + return gitfs.setBranchPrefix(branchPrefix); } // Search // istanbul ignore next export function getFileList(): Promise { - return config.storage.getFileList(); + return gitfs.getFileList(); } // Branch // istanbul ignore next export function branchExists(branchName: string): Promise { - return config.storage.branchExists(branchName); + return gitfs.branchExists(branchName); } // istanbul ignore next export function getAllRenovateBranches( branchPrefix: string ): Promise { - return config.storage.getAllRenovateBranches(branchPrefix); + return gitfs.getAllRenovateBranches(branchPrefix); } // istanbul ignore next export function isBranchStale(branchName: string): Promise { - return config.storage.isBranchStale(branchName); + return gitfs.isBranchStale(branchName); } // istanbul ignore next @@ -537,7 +533,7 @@ export function getFile( filePath: string, branchName?: string ): Promise { - return config.storage.getFile(filePath, branchName); + return gitfs.getFile(filePath, branchName); } // istanbul ignore next @@ -545,17 +541,17 @@ export function deleteBranch( branchName: string, closePr?: boolean ): Promise { - return config.storage.deleteBranch(branchName); + return gitfs.deleteBranch(branchName); } // istanbul ignore next export function getBranchLastCommitTime(branchName: string): Promise { - return config.storage.getBranchLastCommitTime(branchName); + return gitfs.getBranchLastCommitTime(branchName); } // istanbul ignore next -export function getRepoStatus(): Promise { - return config.storage.getRepoStatus(); +export function getRepoStatus(): Promise { + return gitfs.getRepoStatus(); } // istanbul ignore next @@ -566,19 +562,19 @@ export function mergeBranch(branchName: string): Promise { 'Branch protection: Attempting to merge branch when push protection is enabled' ); } - return config.storage.mergeBranch(branchName); + return gitfs.mergeBranch(branchName); } // istanbul ignore next export function commitFiles( commitFilesConfig: CommitFilesConfig ): Promise { - return config.storage.commitFiles(commitFilesConfig); + return gitfs.commitFiles(commitFilesConfig); } // istanbul ignore next export function getCommitMessages(): Promise { - return config.storage.getCommitMessages(); + return gitfs.getCommitMessages(); } async function getClosedPrs(): Promise { @@ -989,7 +985,7 @@ export async function getPrList(): Promise { /* istanbul ignore next */ export async function getPrFiles(pr: Pr): Promise { - return config.storage.getBranchFiles(pr.branchName, pr.targetBranch); + return gitfs.getBranchFiles(pr.branchName, pr.targetBranch); } export async function findPr({ @@ -1139,7 +1135,7 @@ async function getStatusCheck( branchName: string, useCache = true ): Promise { - const branchCommit = await config.storage.getBranchCommit(branchName); + const branchCommit = await gitfs.getBranchCommit(branchName); const url = `repos/${config.repository}/commits/${branchCommit}/statuses`; @@ -1194,7 +1190,7 @@ export async function setBranchStatus({ } logger.debug({ branch: branchName, context, state }, 'Setting branch status'); try { - const branchCommit = await config.storage.getBranchCommit(branchName); + const branchCommit = await gitfs.getBranchCommit(branchName); const url = `repos/${config.repository}/statuses/${branchCommit}`; const renovateToGitHubStateMapping = { green: 'success', diff --git a/lib/platform/github/types.ts b/lib/platform/github/types.ts index 7c8948b3d91a02..c57c56b81032e5 100644 --- a/lib/platform/github/types.ts +++ b/lib/platform/github/types.ts @@ -1,5 +1,4 @@ import { Pr } from '../common'; -import GitStorage from '../git'; // https://developer.github.com/v3/repos/statuses // https://developer.github.com/v3/checks/runs/ @@ -30,7 +29,6 @@ export interface LocalRepoConfig { pushProtection: boolean; prReviewsRequired: boolean; repoForceRebase?: boolean; - storage: GitStorage; parentRepo: string; baseCommitSHA: string | null; forkMode?: boolean; diff --git a/lib/platform/gitlab/index.spec.ts b/lib/platform/gitlab/index.spec.ts index 6341c5760e3dbd..dca6e43fddb744 100644 --- a/lib/platform/gitlab/index.spec.ts +++ b/lib/platform/gitlab/index.spec.ts @@ -14,6 +14,7 @@ import { PR_STATE_OPEN, } from '../../constants/pull-requests'; import { BranchStatus } from '../../types'; +import * as _gitfs from '../../util/gitfs'; import * as _hostRules from '../../util/host-rules'; const gitlabApiHost = 'https://gitlab.com'; @@ -21,7 +22,7 @@ const gitlabApiHost = 'https://gitlab.com'; describe('platform/gitlab', () => { let gitlab: Platform; let hostRules: jest.Mocked; - let GitStorage: jest.Mocked & jest.Mock; + let gitfs: jest.Mocked; beforeEach(async () => { // reset module jest.resetModules(); @@ -30,27 +31,13 @@ describe('platform/gitlab', () => { jest.mock('../../util/host-rules'); jest.mock('delay'); hostRules = require('../../util/host-rules'); - jest.mock('../git'); - GitStorage = require('../git').Storage; - GitStorage.mockImplementation(() => ({ - initRepo: jest.fn(), - cleanRepo: jest.fn(), - getFileList: jest.fn(), - branchExists: jest.fn(() => true), - isBranchStale: jest.fn(() => false), - setBaseBranch: jest.fn(), - getBranchLastCommitTime: jest.fn(), - getAllRenovateBranches: jest.fn(), - getCommitMessages: jest.fn(), - getFile: jest.fn(), - commitFiles: jest.fn(), - mergeBranch: jest.fn(), - deleteBranch: jest.fn(), - getRepoStatus: jest.fn(), - getBranchCommit: jest.fn( - () => '0d9c7726c3d628b7e28af234595cfd20febdbf8e' - ), - })); + jest.mock('../../util/gitfs'); + gitfs = require('../../util/gitfs'); + gitfs.branchExists.mockResolvedValue(true); + gitfs.isBranchStale.mockResolvedValue(true); + gitfs.getBranchCommit.mockResolvedValue( + '0d9c7726c3d628b7e28af234595cfd20febdbf8e' + ); hostRules.find.mockReturnValue({ token: 'abc123', }); @@ -591,11 +578,7 @@ describe('platform/gitlab', () => { }); it('throws repository-changed', async () => { expect.assertions(2); - GitStorage.mockImplementationOnce(() => ({ - initRepo: jest.fn(), - branchExists: jest.fn(() => Promise.resolve(false)), - cleanRepo: jest.fn(), - })); + gitfs.branchExists.mockResolvedValue(false); await initRepo(); await expect(gitlab.getBranchStatus('somebranch', [])).rejects.toThrow( REPOSITORY_CHANGED diff --git a/lib/platform/gitlab/index.ts b/lib/platform/gitlab/index.ts index 4d94406115bbae..0451d9a75e4191 100644 --- a/lib/platform/gitlab/index.ts +++ b/lib/platform/gitlab/index.ts @@ -18,6 +18,7 @@ import { PLATFORM_TYPE_GITLAB } from '../../constants/platforms'; import { PR_STATE_ALL, PR_STATE_OPEN } from '../../constants/pull-requests'; import { logger } from '../../logger'; import { BranchStatus } from '../../types'; +import * as gitfs from '../../util/gitfs'; import * as hostRules from '../../util/host-rules'; import { HttpResponse } from '../../util/http'; import { GitlabHttp, setBaseUrl } from '../../util/http/gitlab'; @@ -38,7 +39,6 @@ import { RepoParams, VulnerabilityAlert, } from '../common'; -import GitStorage, { StatusResult } from '../git'; import { smartTruncate } from '../utils/pr-body'; const gitlabApi = new GitlabHttp(); @@ -58,7 +58,6 @@ type RepoResponse = { }; const defaultConfigFile = configFileNames[0]; let config: { - storage: GitStorage; repository: string; localDir: string; defaultBranch: string; @@ -140,10 +139,7 @@ function urlEscape(str: string): string { } export function cleanRepo(): Promise { - // istanbul ignore if - if (config.storage) { - config.storage.cleanRepo(); - } + gitfs.cleanRepo(); // In theory most of this isn't necessary. In practice.. config = {} as any; return Promise.resolve(); @@ -233,7 +229,7 @@ export async function initRepo({ ) { logger.debug('no http_url_to_repo found. Falling back to old behaviour.'); const { host, protocol } = URL.parse(defaults.endpoint); - url = GitStorage.getUrl({ + url = gitfs.getUrl({ protocol: protocol.slice(0, -1) as any, auth: 'oauth2:' + opts.token, host, @@ -245,8 +241,7 @@ export async function initRepo({ repoUrl.auth = 'oauth2:' + opts.token; url = URL.format(repoUrl); } - config.storage = new GitStorage(); - await config.storage.initRepo({ + await gitfs.initRepo({ ...config, url, gitAuthorName: global.gitAuthor?.name, @@ -288,26 +283,26 @@ export async function setBaseBranch( ): Promise { logger.debug(`Setting baseBranch to ${branchName}`); config.baseBranch = branchName; - const baseBranchSha = await config.storage.setBaseBranch(branchName); + const baseBranchSha = await gitfs.setBaseBranch(branchName); return baseBranchSha; } export /* istanbul ignore next */ function setBranchPrefix( branchPrefix: string ): Promise { - return config.storage.setBranchPrefix(branchPrefix); + return gitfs.setBranchPrefix(branchPrefix); } // Search // Get full file list export function getFileList(): Promise { - return config.storage.getFileList(); + return gitfs.getFileList(); } // Returns true if branch exists, otherwise false export function branchExists(branchName: string): Promise { - return config.storage.branchExists(branchName); + return gitfs.branchExists(branchName); } type BranchState = 'pending' | 'running' | 'success' | 'failed' | 'canceled'; @@ -322,7 +317,7 @@ async function getStatus( branchName: string, useCache = true ): Promise { - const branchSha = await config.storage.getBranchCommit(branchName); + const branchSha = await gitfs.getBranchCommit(branchName); const url = `projects/${config.repository}/repository/commits/${branchSha}/statuses`; return ( @@ -629,25 +624,25 @@ export async function getBranchPr(branchName: string): Promise { export function getAllRenovateBranches( branchPrefix: string ): Promise { - return config.storage.getAllRenovateBranches(branchPrefix); + return gitfs.getAllRenovateBranches(branchPrefix); } export function isBranchStale(branchName: string): Promise { - return config.storage.isBranchStale(branchName); + return gitfs.isBranchStale(branchName); } // istanbul ignore next export function commitFiles( commitFilesConfig: CommitFilesConfig ): Promise { - return config.storage.commitFiles(commitFilesConfig); + return gitfs.commitFiles(commitFilesConfig); } export function getFile( filePath: string, branchName?: string ): Promise { - return config.storage.getFile(filePath, branchName); + return gitfs.getFile(filePath, branchName); } export async function deleteBranch( @@ -662,20 +657,20 @@ export async function deleteBranch( await closePr(pr.number); } } - return config.storage.deleteBranch(branchName); + return gitfs.deleteBranch(branchName); } export function mergeBranch(branchName: string): Promise { - return config.storage.mergeBranch(branchName); + return gitfs.mergeBranch(branchName); } export function getBranchLastCommitTime(branchName: string): Promise { - return config.storage.getBranchLastCommitTime(branchName); + return gitfs.getBranchLastCommitTime(branchName); } // istanbul ignore next -export function getRepoStatus(): Promise { - return config.storage.getRepoStatus(); +export function getRepoStatus(): Promise { + return gitfs.getRepoStatus(); } export async function getBranchStatusCheck( @@ -701,7 +696,7 @@ export async function setBranchStatus({ url: targetUrl, }: BranchStatusConfig): Promise { // First, get the branch commit SHA - const branchSha = await config.storage.getBranchCommit(branchName); + const branchSha = await gitfs.getBranchCommit(branchName); // Now, check the statuses for that commit const url = `projects/${config.repository}/statuses/${branchSha}`; let state = 'success'; @@ -1085,7 +1080,7 @@ export async function getPrList(): Promise { /* istanbul ignore next */ export async function getPrFiles(pr: Pr): Promise { - return config.storage.getBranchFiles(pr.branchName, pr.targetBranch); + return gitfs.getBranchFiles(pr.branchName, pr.targetBranch); } function matchesState(state: string, desiredState: string): boolean { @@ -1114,7 +1109,7 @@ export async function findPr({ } export function getCommitMessages(): Promise { - return config.storage.getCommitMessages(); + return gitfs.getCommitMessages(); } export function getVulnerabilityAlerts(): Promise { diff --git a/lib/platform/index.ts b/lib/platform/index.ts index 3e8eb6e41284f0..a9f4a6d265b7b0 100644 --- a/lib/platform/index.ts +++ b/lib/platform/index.ts @@ -3,10 +3,10 @@ import addrs from 'email-addresses'; import { RenovateConfig } from '../config/common'; import { PLATFORM_NOT_FOUND } from '../constants/error-messages'; import { logger } from '../logger'; +import { setPrivateKey } from '../util/gitfs'; import * as hostRules from '../util/host-rules'; import platforms from './api.generated'; import { Platform } from './common'; -import { setPrivateKey } from './git/private-key'; export * from './common'; diff --git a/lib/platform/git/__snapshots__/index.spec.ts.snap b/lib/util/gitfs/git/__snapshots__/index.spec.ts.snap similarity index 100% rename from lib/platform/git/__snapshots__/index.spec.ts.snap rename to lib/util/gitfs/git/__snapshots__/index.spec.ts.snap diff --git a/lib/platform/git/index.spec.ts b/lib/util/gitfs/git/index.spec.ts similarity index 74% rename from lib/platform/git/index.spec.ts rename to lib/util/gitfs/git/index.spec.ts index 6aa6a32cdd5aaa..3e9e8af2173cdc 100644 --- a/lib/platform/git/index.spec.ts +++ b/lib/util/gitfs/git/index.spec.ts @@ -1,12 +1,11 @@ import fs from 'fs-extra'; import Git from 'simple-git/promise'; import tmp from 'tmp-promise'; -import GitStorage from '.'; +import * as gitfs from '.'; describe('platform/git', () => { jest.setTimeout(15000); - const git = new GitStorage(); const masterCommitDate = new Date(); masterCommitDate.setMilliseconds(0); let base: tmp.DirectoryResult; @@ -47,7 +46,7 @@ describe('platform/git', () => { const repo = Git(origin.path); await repo.clone(base.path, '.', ['--bare']); tmpDir = await tmp.dir({ unsafeCleanup: true }); - await git.initRepo({ + await gitfs.initRepo({ localDir: tmpDir.path, url: origin.path, extraCloneOpts: { @@ -61,7 +60,7 @@ describe('platform/git', () => { afterEach(async () => { await tmpDir.cleanup(); await origin.cleanup(); - git.cleanRepo(); + gitfs.cleanRepo(); }); afterAll(async () => { @@ -70,45 +69,45 @@ describe('platform/git', () => { describe('setBaseBranch(branchName)', () => { it('sets the base branch as master', async () => { - await expect(git.setBaseBranch('master')).resolves.not.toThrow(); + await expect(gitfs.setBaseBranch('master')).resolves.not.toThrow(); }); it('sets non-master base branch', async () => { - await expect(git.setBaseBranch('develop')).resolves.not.toThrow(); + await expect(gitfs.setBaseBranch('develop')).resolves.not.toThrow(); }); it('should throw if branch does not exist', async () => { - await expect(git.setBaseBranch('not_found')).rejects.toMatchSnapshot(); + await expect(gitfs.setBaseBranch('not_found')).rejects.toMatchSnapshot(); }); }); describe('getFileList()', () => { it('should return the correct files', async () => { - expect(await git.getFileList()).toMatchSnapshot(); + expect(await gitfs.getFileList()).toMatchSnapshot(); }); it('should exclude submodules', async () => { const repo = Git(base.path).silent(true); await repo.submoduleAdd(base.path, 'submodule'); await repo.commit('Add submodule'); - await git.initRepo({ + await gitfs.initRepo({ localDir: tmpDir.path, url: base.path, }); expect(await fs.exists(tmpDir.path + '/.gitmodules')).toBeTruthy(); - expect(await git.getFileList()).toMatchSnapshot(); + expect(await gitfs.getFileList()).toMatchSnapshot(); await repo.reset(['--hard', 'HEAD^']); }); }); describe('branchExists(branchName)', () => { it('should return true if found', async () => { - expect(await git.branchExists('renovate/future_branch')).toBe(true); - expect(await git.branchExists('renovate/future_branch')).toBe(true); // should come from cache + expect(await gitfs.branchExists('renovate/future_branch')).toBe(true); + expect(await gitfs.branchExists('renovate/future_branch')).toBe(true); // should come from cache }); it('should return false if not found', async () => { - expect(await git.branchExists('not_found')).toBe(false); + expect(await gitfs.branchExists('not_found')).toBe(false); }); }); describe('getAllRenovateBranches()', () => { it('should return all renovate branches', async () => { - await git.setBranchPrefix('renovate/'); - const res = await git.getAllRenovateBranches('renovate/'); + await gitfs.setBranchPrefix('renovate/'); + const res = await gitfs.getAllRenovateBranches('renovate/'); expect(res).toContain('renovate/past_branch'); expect(res).toContain('renovate/future_branch'); expect(res).not.toContain('master'); @@ -116,68 +115,72 @@ describe('platform/git', () => { }); describe('isBranchStale()', () => { it('should return false if same SHA as master', async () => { - expect(await git.isBranchStale('renovate/future_branch')).toBe(false); + expect(await gitfs.isBranchStale('renovate/future_branch')).toBe(false); }); it('should return true if SHA different from master', async () => { - expect(await git.isBranchStale('renovate/past_branch')).toBe(true); + expect(await gitfs.isBranchStale('renovate/past_branch')).toBe(true); }); it('should throw if branch does not exist', async () => { - await expect(git.isBranchStale('not_found')).rejects.toMatchSnapshot(); + await expect(gitfs.isBranchStale('not_found')).rejects.toMatchSnapshot(); }); }); describe('getBranchCommit(branchName)', () => { it('should return same value for equal refs', async () => { - const hex = await git.getBranchCommit('renovate/past_branch'); - expect(hex).toBe(await git.getBranchCommit('master~1')); + const hex = await gitfs.getBranchCommit('renovate/past_branch'); + expect(hex).toBe(await gitfs.getBranchCommit('master~1')); expect(hex).toHaveLength(40); }); it('should throw if branch does not exist', async () => { - await expect(git.getBranchCommit('not_found')).rejects.toMatchSnapshot(); + await expect( + gitfs.getBranchCommit('not_found') + ).rejects.toMatchSnapshot(); }); }); describe('createBranch(branchName, sha)', () => { it('resets existing branch', async () => { - const hex = await git.getBranchCommit('renovate/past_branch'); - expect(await git.getBranchCommit('renovate/future_branch')).not.toBe(hex); - await git.createBranch('renovate/future_branch', hex); - expect(await git.getBranchCommit('renovate/future_branch')).toBe(hex); + const hex = await gitfs.getBranchCommit('renovate/past_branch'); + expect(await gitfs.getBranchCommit('renovate/future_branch')).not.toBe( + hex + ); + await gitfs.createBranch('renovate/future_branch', hex); + expect(await gitfs.getBranchCommit('renovate/future_branch')).toBe(hex); }); }); describe('getBranchFiles(branchName, baseBranchName?)', () => { it('detects changed files', async () => { - const hex = await git.getBranchCommit('master'); - await git.createBranch('renovate/branch_with_changes', hex); + const hex = await gitfs.getBranchCommit('master'); + await gitfs.createBranch('renovate/branch_with_changes', hex); const file = { name: 'some-new-file', contents: 'some new-contents', }; - await git.commitFiles({ + await gitfs.commitFiles({ branchName: 'renovate/branch_with_changes', files: [file], message: 'Create something', }); - const branchFiles = await git.getBranchFiles( + const branchFiles = await gitfs.getBranchFiles( 'renovate/branch_with_changes', 'master' ); expect(branchFiles).toMatchSnapshot(); }); it('detects changed files compared to current base branch', async () => { - const hex = await git.getBranchCommit('master'); - await git.createBranch('renovate/branch_with_changes', hex); + const hex = await gitfs.getBranchCommit('master'); + await gitfs.createBranch('renovate/branch_with_changes', hex); const file = { name: 'some-new-file', contents: 'some new-contents', }; - await git.commitFiles({ + await gitfs.commitFiles({ branchName: 'renovate/branch_with_changes', files: [file], message: 'Create something', }); - const branchFiles = await git.getBranchFiles( + const branchFiles = await gitfs.getBranchFiles( 'renovate/branch_with_changes' ); expect(branchFiles).toMatchSnapshot(); @@ -186,8 +189,8 @@ describe('platform/git', () => { describe('mergeBranch(branchName)', () => { it('should perform a branch merge', async () => { - await git.setBranchPrefix('renovate/'); - await git.mergeBranch('renovate/future_branch'); + await gitfs.setBranchPrefix('renovate/'); + await gitfs.mergeBranch('renovate/future_branch'); const merged = await Git(origin.path).branch([ '--verbose', '--merged', @@ -196,38 +199,38 @@ describe('platform/git', () => { expect(merged.all).toContain('renovate/future_branch'); }); it('should throw if branch merge throws', async () => { - await expect(git.mergeBranch('not_found')).rejects.toThrow(); + await expect(gitfs.mergeBranch('not_found')).rejects.toThrow(); }); }); describe('deleteBranch(branchName)', () => { it('should send delete', async () => { - await git.deleteBranch('renovate/past_branch'); + await gitfs.deleteBranch('renovate/past_branch'); const branches = await Git(origin.path).branch({}); expect(branches.all).not.toContain('renovate/past_branch'); }); }); describe('getBranchLastCommitTime', () => { it('should return a Date', async () => { - const time = await git.getBranchLastCommitTime('master'); + const time = await gitfs.getBranchLastCommitTime('master'); expect(time).toEqual(masterCommitDate); }); it('handles error', async () => { - const res = await git.getBranchLastCommitTime('some-branch'); + const res = await gitfs.getBranchLastCommitTime('some-branch'); expect(res).toBeDefined(); }); }); describe('getFile(filePath, branchName)', () => { it('gets the file', async () => { - const res = await git.getFile('master_file'); + const res = await gitfs.getFile('master_file'); expect(res).toBe('master'); }); it('short cuts 404', async () => { - const res = await git.getFile('some-missing-path'); + const res = await gitfs.getFile('some-missing-path'); expect(res).toBeNull(); }); it('returns null for 404', async () => { await expect( - git.getFile('some-path', 'some-branch') + gitfs.getFile('some-path', 'some-branch') ).rejects.toMatchSnapshot(); }); }); @@ -237,7 +240,7 @@ describe('platform/git', () => { name: 'some-new-file', contents: 'some new-contents', }; - const commit = await git.commitFiles({ + const commit = await gitfs.commitFiles({ branchName: 'renovate/past_branch', files: [file], message: 'Create something', @@ -249,7 +252,7 @@ describe('platform/git', () => { name: '|delete|', contents: 'file_to_delete', }; - const commit = await git.commitFiles({ + const commit = await gitfs.commitFiles({ branchName: 'renovate/something', files: [file], message: 'Delete something', @@ -267,7 +270,7 @@ describe('platform/git', () => { contents: 'other updated content', }, ]; - const commit = await git.commitFiles({ + const commit = await gitfs.commitFiles({ branchName: 'renovate/something', files, message: 'Update something', @@ -281,7 +284,7 @@ describe('platform/git', () => { contents: 'some content', }, ]; - const commit = await git.commitFiles({ + const commit = await gitfs.commitFiles({ branchName: 'renovate/something', files, message: 'Update something', @@ -297,7 +300,7 @@ describe('platform/git', () => { `refs/heads/${branchName}:refs/remotes/origin/${branchName}`, ]); const files = []; - const commit = await git.commitFiles({ + const commit = await gitfs.commitFiles({ branchName, files, message: 'Update something', @@ -308,12 +311,12 @@ describe('platform/git', () => { describe('getCommitMessages()', () => { it('returns commit messages', async () => { - expect(await git.getCommitMessages()).toMatchSnapshot(); + expect(await gitfs.getCommitMessages()).toMatchSnapshot(); }); }); describe('Storage.getUrl()', () => { - const getUrl = GitStorage.getUrl; + const getUrl = gitfs.getUrl; it('returns https url', () => { expect( getUrl({ @@ -353,22 +356,22 @@ describe('platform/git', () => { await repo.commit('past message2'); await repo.checkout('master'); - expect(await git.branchExists('test')).toBeFalsy(); + expect(await gitfs.branchExists('test')).toBeFalsy(); - expect(await git.getCommitMessages()).toMatchSnapshot(); + expect(await gitfs.getCommitMessages()).toMatchSnapshot(); - await git.setBaseBranch('develop'); + await gitfs.setBaseBranch('develop'); - await git.initRepo({ + await gitfs.initRepo({ localDir: tmpDir.path, url: base.path, }); - expect(await git.branchExists('test')).toBeTruthy(); + expect(await gitfs.branchExists('test')).toBeTruthy(); - await git.setBaseBranch('test'); + await gitfs.setBaseBranch('test'); - const msg = await git.getCommitMessages(); + const msg = await gitfs.getCommitMessages(); expect(msg).toMatchSnapshot(); expect(msg).toContain('past message2'); }); @@ -381,16 +384,16 @@ describe('platform/git', () => { await repo.commit('past message2'); await repo.checkout('master'); - await git.initRepo({ + await gitfs.initRepo({ localDir: tmpDir.path, url: base.path, }); - await git.setBranchPrefix('renovate/'); - expect(await git.branchExists('renovate/test')).toBe(true); - const cid = await git.getBranchCommit('renovate/test'); + await gitfs.setBranchPrefix('renovate/'); + expect(await gitfs.branchExists('renovate/test')).toBe(true); + const cid = await gitfs.getBranchCommit('renovate/test'); - await git.initRepo({ + await gitfs.initRepo({ localDir: tmpDir.path, url: base.path, }); @@ -398,9 +401,9 @@ describe('platform/git', () => { await repo.checkout('renovate/test'); await repo.commit('past message3', ['--amend']); - await git.setBranchPrefix('renovate/'); - expect(await git.branchExists('renovate/test')).toBe(true); - expect(await git.getBranchCommit('renovate/test')).not.toEqual(cid); + await gitfs.setBranchPrefix('renovate/'); + expect(await gitfs.branchExists('renovate/test')).toBe(true); + expect(await gitfs.getBranchCommit('renovate/test')).not.toEqual(cid); }); it('should fail clone ssh submodule', async () => { @@ -419,7 +422,7 @@ describe('platform/git', () => { 'test', ]); await repo.commit('Add submodule'); - await git.initRepo({ + await gitfs.initRepo({ localDir: tmpDir.path, url: base.path, }); diff --git a/lib/util/gitfs/git/index.ts b/lib/util/gitfs/git/index.ts new file mode 100644 index 00000000000000..eb6a86555e45da --- /dev/null +++ b/lib/util/gitfs/git/index.ts @@ -0,0 +1,599 @@ +import { join } from 'path'; +import URL from 'url'; +import fs from 'fs-extra'; +import Git from 'simple-git/promise'; +import { + CONFIG_VALIDATION, + REPOSITORY_CHANGED, + REPOSITORY_EMPTY, + REPOSITORY_TEMPORARY_ERROR, + SYSTEM_INSUFFICIENT_DISK_SPACE, +} from '../../../constants/error-messages'; +import { logger } from '../../../logger'; +import { CommitFilesConfig } from '../../../platform/common'; +import { ExternalHostError } from '../../../types/errors/external-host-error'; +import * as limits from '../../../workers/global/limits'; +import { writePrivateKey } from './private-key'; + +declare module 'fs-extra' { + export function exists(pathLike: string): Promise; +} + +export type StatusResult = Git.StatusResult; + +export type DiffResult = Git.DiffResult; + +interface StorageConfig { + localDir: string; + baseBranch?: string; + url: string; + extraCloneOpts?: Git.Options; + gitAuthorName?: string; + gitAuthorEmail?: string; +} + +interface LocalConfig extends StorageConfig { + baseBranch: string; + baseBranchSha: string; + branchExists: Record; + branchPrefix: string; +} + +// istanbul ignore next +function checkForPlatformFailure(err: Error): void { + if (process.env.NODE_ENV === 'test') { + return; + } + const platformFailureStrings = [ + 'remote: Invalid username or password', + 'gnutls_handshake() failed', + 'The requested URL returned error: 5', + 'The remote end hung up unexpectedly', + 'access denied or repository not exported', + 'Could not write new index file', + 'Failed to connect to', + 'Connection timed out', + ]; + for (const errorStr of platformFailureStrings) { + if (err.message.includes(errorStr)) { + throw new ExternalHostError(err, 'git'); + } + } +} + +function localName(branchName: string): string { + return branchName.replace(/^origin\//, ''); +} + +function throwBaseBranchValidationError(branchName: string): never { + const error = new Error(CONFIG_VALIDATION); + error.validationError = 'baseBranch not found'; + error.validationMessage = + 'The following configured baseBranch could not be found: ' + branchName; + throw error; +} + +async function isDirectory(dir: string): Promise { + try { + return (await fs.stat(dir)).isDirectory(); + } catch (err) { + return false; + } +} + +async function getDefaultBranch(git: Git.SimpleGit): Promise { + // see https://stackoverflow.com/a/44750379/1438522 + try { + const res = await git.raw(['symbolic-ref', 'refs/remotes/origin/HEAD']); + return res.replace('refs/remotes/origin/', '').trim(); + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + if ( + err.message.startsWith( + 'fatal: ref refs/remotes/origin/HEAD is not a symbolic ref' + ) + ) { + throw new Error(REPOSITORY_EMPTY); + } + throw err; + } +} + +let config: LocalConfig = {} as any; + +let git: Git.SimpleGit | undefined; + +let cwd: string | undefined; + +let privateKeySet = false; + +async function resetToBranch(branchName: string): Promise { + logger.debug(`resetToBranch(${branchName})`); + await git.raw(['reset', '--hard']); + await git.checkout(branchName); + await git.raw(['reset', '--hard', 'origin/' + branchName]); + await git.raw(['clean', '-fd']); +} + +async function deleteLocalBranch(branchName: string): Promise { + await git.branch(['-D', branchName]); +} + +async function cleanLocalBranches(): Promise { + const existingBranches = (await git.raw(['branch'])) + .split('\n') + .map((branch) => branch.trim()) + .filter((branch) => branch.length) + .filter((branch) => !branch.startsWith('* ')); + logger.debug({ existingBranches }); + for (const branchName of existingBranches) { + await deleteLocalBranch(branchName); + } +} + +export async function getSubmodules(): Promise { + return ( + (await git.raw([ + 'config', + '--file', + '.gitmodules', + '--get-regexp', + 'path', + ])) || '' + ) + .trim() + .split(/[\n\s]/) + .filter((_e: string, i: number) => i % 2); +} + +export function isInitialized(): boolean { + return !!git; +} + +export function cleanRepo(): void { + if (isInitialized()) { + // no-op + } +} + +export async function initRepo(args: StorageConfig): Promise { + cleanRepo(); + + config = { ...args } as any; + const newConfig: LocalConfig = config; + + cwd = newConfig.localDir; + const newCwd = cwd; + + newConfig.branchExists = {}; + logger.debug('Initializing git repository into ' + newCwd); + const gitHead = join(newCwd, '.git/HEAD'); + let clone = true; + + if (await fs.exists(gitHead)) { + try { + git = Git(newCwd).silent(true); + await git.raw(['remote', 'set-url', 'origin', newConfig.url]); + const fetchStart = Date.now(); + await git.fetch(['--depth=10']); + newConfig.baseBranch = + newConfig.baseBranch || (await getDefaultBranch(git)); + await resetToBranch(newConfig.baseBranch); + await cleanLocalBranches(); + await git.raw(['remote', 'prune', 'origin']); + const durationMs = Math.round(Date.now() - fetchStart); + logger.debug({ durationMs }, 'git fetch completed'); + clone = false; + } catch (err) /* istanbul ignore next */ { + logger.error({ err }, 'git fetch error'); + } + } + if (clone) { + await fs.emptyDir(newCwd); + git = Git(newCwd).silent(true); + const cloneStart = Date.now(); + try { + // clone only the default branch + let opts = ['--depth=2']; + if (newConfig.extraCloneOpts) { + opts = opts.concat( + Object.entries(newConfig.extraCloneOpts).map((e) => `${e[0]}=${e[1]}`) + ); + } + await git.clone(newConfig.url, '.', opts); + } catch (err) /* istanbul ignore next */ { + logger.debug({ err }, 'git clone error'); + if (err.message?.includes('write error: No space left on device')) { + throw new Error(SYSTEM_INSUFFICIENT_DISK_SPACE); + } + throw new ExternalHostError(err, 'git'); + } + const durationMs = Math.round(Date.now() - cloneStart); + logger.debug({ durationMs }, 'git clone completed'); + } + const submodules = await getSubmodules(); + for (const submodule of submodules) { + try { + logger.debug(`Cloning git submodule at ${submodule}`); + await git.submoduleUpdate(['--init', '--', submodule]); + } catch (err) { + logger.warn(`Unable to initialise git submodule at ${submodule}`); + } + } + try { + const latestCommitDate = (await git.log({ n: 1 })).latest.date; + logger.debug({ latestCommitDate }, 'latest commit'); + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + if (err.message.includes('does not have any commits yet')) { + throw new Error(REPOSITORY_EMPTY); + } + logger.warn({ err }, 'Cannot retrieve latest commit date'); + } + try { + const { gitAuthorName, gitAuthorEmail } = args; + if (gitAuthorName) { + logger.debug({ gitAuthorName }, 'Setting git author name'); + await git.raw(['config', 'user.name', gitAuthorName]); + } + if (gitAuthorEmail) { + logger.debug({ gitAuthorEmail }, 'Setting git author email'); + await git.raw(['config', 'user.email', gitAuthorEmail]); + } + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + logger.debug({ err }, 'Error setting git author config'); + throw new Error(REPOSITORY_TEMPORARY_ERROR); + } + + newConfig.baseBranch = newConfig.baseBranch || (await getDefaultBranch(git)); +} + +// istanbul ignore next +export async function getRepoStatus(): Promise { + return git.status(); +} + +export async function createBranch( + branchName: string, + sha: string +): Promise { + logger.debug(`createBranch(${branchName})`); + await git.reset('hard'); + await git.raw(['clean', '-fd']); + await git.checkout(['-B', branchName, sha]); + await git.push('origin', branchName, { '--force': true }); + config.branchExists[branchName] = true; +} + +export async function branchExists(branchName: string): Promise { + // First check cache + if (config.branchExists[branchName] !== undefined) { + return config.branchExists[branchName]; + } + if (!branchName.startsWith(config.branchPrefix)) { + // fetch the branch only if it's not part of the existing branchPrefix + try { + await git.raw(['remote', 'set-branches', '--add', 'origin', branchName]); + await git.fetch(['origin', branchName, '--depth=2']); + } catch (err) { + checkForPlatformFailure(err); + } + } + try { + await git.raw(['show-branch', 'origin/' + branchName]); + config.branchExists[branchName] = true; + return true; + } catch (err) { + checkForPlatformFailure(err); + config.branchExists[branchName] = false; + return false; + } +} + +// Return the commit SHA for a branch +export async function getBranchCommit(branchName: string): Promise { + if (!(await branchExists(branchName))) { + throw Error( + 'Cannot fetch commit for branch that does not exist: ' + branchName + ); + } + const res = await git.revparse(['origin/' + branchName]); + return res.trim(); +} + +export async function getCommitMessages(): Promise { + logger.debug('getCommitMessages'); + const res = await git.log({ + n: 10, + format: { message: '%s' }, + }); + return res.all.map((commit) => commit.message); +} + +export async function setBaseBranch(branchName: string): Promise { + if (branchName) { + if (!(await branchExists(branchName))) { + throwBaseBranchValidationError(branchName); + } + logger.debug(`Setting baseBranch to ${branchName}`); + config.baseBranch = branchName; + try { + if (branchName !== 'master') { + config.baseBranchSha = ( + await git.raw(['rev-parse', 'origin/' + branchName]) + ).trim(); + } + await git.checkout([branchName, '-f']); + await git.reset('hard'); + const latestCommitDate = (await git.log({ n: 1 })).latest.date; + logger.debug({ branchName, latestCommitDate }, 'latest commit'); + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + if ( + err.message.includes( + 'unknown revision or path not in the working tree' + ) || + err.message.includes('did not match any file(s) known to git') + ) { + throwBaseBranchValidationError(branchName); + } + throw err; + } + } + return ( + config.baseBranchSha || + (await git.raw(['rev-parse', 'origin/master'])).trim() + ); +} + +/* + * When we initially clone, we clone only the default branch so how no knowledge of other branches existing. + * By calling this function once the repo's branchPrefix is known, we can fetch all of Renovate's branches in one command. + */ +export async function setBranchPrefix(branchPrefix: string): Promise { + logger.debug('Setting branchPrefix: ' + branchPrefix); + config.branchPrefix = branchPrefix; + const ref = `refs/heads/${branchPrefix}*:refs/remotes/origin/${branchPrefix}*`; + try { + await git.fetch(['origin', ref, '--depth=2', '--force']); + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + throw err; + } +} + +export async function getFileList(): Promise { + const branch = config.baseBranch; + const submodules = await getSubmodules(); + const files: string = await git.raw(['ls-tree', '-r', branch]); + // istanbul ignore if + if (!files) { + return []; + } + return files + .split('\n') + .filter(Boolean) + .filter((line) => line.startsWith('100')) + .map((line) => line.split(/\t/).pop()) + .filter((file: string) => + submodules.every((submodule: string) => !file.startsWith(submodule)) + ); +} + +export async function getAllRenovateBranches( + branchPrefix: string +): Promise { + const branches = await git.branch(['--remotes', '--verbose']); + return branches.all + .map(localName) + .filter((branchName) => branchName.startsWith(branchPrefix)); +} + +export async function isBranchStale(branchName: string): Promise { + if (!(await branchExists(branchName))) { + throw Error( + 'Cannot check staleness for branch that does not exist: ' + branchName + ); + } + const branches = await git.branch([ + '--remotes', + '--verbose', + '--contains', + config.baseBranchSha || `origin/${config.baseBranch}`, + ]); + return !branches.all.map(localName).includes(branchName); +} + +export async function deleteBranch(branchName: string): Promise { + try { + await git.raw(['push', '--delete', 'origin', branchName]); + logger.debug({ branchName }, 'Deleted remote branch'); + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + logger.debug({ branchName }, 'No remote branch to delete'); + } + try { + await deleteLocalBranch(branchName); + // istanbul ignore next + logger.debug({ branchName }, 'Deleted local branch'); + } catch (err) { + checkForPlatformFailure(err); + logger.debug({ branchName }, 'No local branch to delete'); + } + config.branchExists[branchName] = false; +} + +export async function mergeBranch(branchName: string): Promise { + await git.reset('hard'); + await git.checkout(['-B', branchName, 'origin/' + branchName]); + await git.checkout(config.baseBranch); + await git.merge(['--ff-only', branchName]); + await git.push('origin', config.baseBranch); + limits.incrementLimit('prCommitsPerRunLimit'); +} + +export async function getBranchLastCommitTime( + branchName: string +): Promise { + try { + const time = await git.show(['-s', '--format=%ai', 'origin/' + branchName]); + return new Date(Date.parse(time)); + } catch (err) { + checkForPlatformFailure(err); + return new Date(); + } +} + +export async function getBranchFiles( + branchName: string, + baseBranchName?: string +): Promise { + try { + const diff = await git.diffSummary([ + branchName, + baseBranchName || config.baseBranch, + ]); + return diff.files.map((file) => file.file); + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + return null; + } +} + +export async function getFile( + filePath: string, + branchName?: string +): Promise { + if (branchName) { + const exists = await branchExists(branchName); + if (!exists) { + logger.debug({ branchName }, 'branch no longer exists - aborting'); + throw new Error(REPOSITORY_CHANGED); + } + } + try { + const content = await git.show([ + 'origin/' + (branchName || config.baseBranch) + ':' + filePath, + ]); + return content; + } catch (err) { + checkForPlatformFailure(err); + return null; + } +} + +export async function hasDiff(branchName: string): Promise { + try { + return (await git.diff(['HEAD', branchName])) !== ''; + } catch (err) { + return true; + } +} + +export async function commitFiles({ + branchName, + files, + message, + force = false, +}: CommitFilesConfig): Promise { + logger.debug(`Committing files to branch ${branchName}`); + if (!privateKeySet) { + await writePrivateKey(cwd); + privateKeySet = true; + } + try { + await git.reset('hard'); + await git.raw(['clean', '-fd']); + await git.checkout(['-B', branchName, 'origin/' + config.baseBranch]); + const fileNames = []; + const deleted = []; + for (const file of files) { + // istanbul ignore if + if (file.name === '|delete|') { + deleted.push(file.contents); + } else if (await isDirectory(join(cwd, file.name))) { + fileNames.push(file.name); + await git.add(file.name); + } else { + fileNames.push(file.name); + let contents; + // istanbul ignore else + if (typeof file.contents === 'string') { + contents = Buffer.from(file.contents); + } else { + contents = file.contents; + } + await fs.outputFile(join(cwd, file.name), contents); + } + } + // istanbul ignore if + if (fileNames.length === 1 && fileNames[0] === 'renovate.json') { + fileNames.unshift('-f'); + } + if (fileNames.length) { + await git.add(fileNames); + } + if (deleted.length) { + for (const f of deleted) { + try { + await git.rm([f]); + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + logger.debug({ err }, 'Cannot delete ' + f); + } + } + } + const commitRes = await git.commit(message, [], { + '--no-verify': true, + }); + const commit = commitRes?.commit || 'unknown'; + if (!force && !(await hasDiff(`origin/${branchName}`))) { + logger.debug( + { branchName, fileNames }, + 'No file changes detected. Skipping commit' + ); + return null; + } + await git.push('origin', `${branchName}:${branchName}`, { + '--force': true, + '-u': true, + '--no-verify': true, + }); + // Fetch it after create + const ref = `refs/heads/${branchName}:refs/remotes/origin/${branchName}`; + await git.fetch(['origin', ref, '--depth=2', '--force']); + config.branchExists[branchName] = true; + limits.incrementLimit('prCommitsPerRunLimit'); + return commit; + } catch (err) /* istanbul ignore next */ { + checkForPlatformFailure(err); + logger.debug({ err }, 'Error commiting files'); + throw new Error(REPOSITORY_CHANGED); + } +} + +export function getUrl({ + protocol, + auth, + hostname, + host, + repository, +}: { + protocol?: 'ssh' | 'http' | 'https'; + auth?: string; + hostname?: string; + host?: string; + repository: string; +}): string { + if (protocol === 'ssh') { + return `git@${hostname}:${repository}.git`; + } + return URL.format({ + protocol: protocol || 'https', + auth, + hostname, + host, + pathname: repository + '.git', + }); +} diff --git a/lib/platform/git/private-key.spec.ts b/lib/util/gitfs/git/private-key.spec.ts similarity index 89% rename from lib/platform/git/private-key.spec.ts rename to lib/util/gitfs/git/private-key.spec.ts index 7a8cdf63357133..e99061784d5228 100644 --- a/lib/platform/git/private-key.spec.ts +++ b/lib/util/gitfs/git/private-key.spec.ts @@ -1,9 +1,9 @@ -import { getName, mocked } from '../../../test/util'; -import * as exec_ from '../../util/exec'; +import { getName, mocked } from '../../../../test/util'; +import * as exec_ from '../../exec'; import { setPrivateKey, writePrivateKey } from './private-key'; jest.mock('fs-extra'); -jest.mock('../../util/exec'); +jest.mock('../../exec'); const exec = mocked(exec_); diff --git a/lib/platform/git/private-key.ts b/lib/util/gitfs/git/private-key.ts similarity index 88% rename from lib/platform/git/private-key.ts rename to lib/util/gitfs/git/private-key.ts index 680134f4b5632a..f4f6ecd6a76579 100644 --- a/lib/platform/git/private-key.ts +++ b/lib/util/gitfs/git/private-key.ts @@ -1,9 +1,9 @@ import os from 'os'; import path from 'path'; import fs from 'fs-extra'; -import { PLATFORM_GPG_FAILED } from '../../constants/error-messages'; -import { logger } from '../../logger'; -import { exec } from '../../util/exec'; +import { PLATFORM_GPG_FAILED } from '../../../constants/error-messages'; +import { logger } from '../../../logger'; +import { exec } from '../../exec'; let gitPrivateKey: string; let keyId: string; diff --git a/lib/util/gitfs/index.ts b/lib/util/gitfs/index.ts index c6a897d250b7ed..dc681c92741c9e 100644 --- a/lib/util/gitfs/index.ts +++ b/lib/util/gitfs/index.ts @@ -1 +1,3 @@ export * from './fs'; +export * from './git'; +export * from './git/private-key'; diff --git a/lib/workers/branch/index.spec.ts b/lib/workers/branch/index.spec.ts index 83aadfb209a64d..85bd692ee0b0e8 100644 --- a/lib/workers/branch/index.spec.ts +++ b/lib/workers/branch/index.spec.ts @@ -11,8 +11,8 @@ import { } from '../../constants/pull-requests'; import * as _npmPostExtract from '../../manager/npm/post-update'; import { File } from '../../platform'; -import { StatusResult } from '../../platform/git'; import * as _exec from '../../util/exec'; +import { StatusResult } from '../../util/gitfs'; import { BranchConfig, PrResult } from '../common'; import * as _prWorker from '../pr'; import * as _automerge from './automerge';