From d70b8c1f2fa999be9684a8ca5da3dda8cd2392ab Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 29 Jun 2020 15:51:22 +0200 Subject: [PATCH] feat: repositoryCache (#6589) Co-authored-by: Michael Kriese --- docs/usage/self-hosted-configuration.md | 6 ++ lib/config/common.ts | 4 ++ lib/config/definitions.ts | 9 +++ lib/util/cache/repository/index.spec.ts | 39 +++++++++++++ lib/util/cache/repository/index.ts | 58 +++++++++++++++++++ lib/workers/repository/finalise/index.ts | 2 + lib/workers/repository/init/config.ts | 7 +++ lib/workers/repository/init/index.ts | 2 + .../__snapshots__/extract-update.spec.ts.snap | 4 +- .../repository/process/extract-update.spec.ts | 37 +++++++++++- .../repository/process/extract-update.ts | 34 ++++++++++- package.json | 1 + yarn.lock | 5 ++ 13 files changed, 205 insertions(+), 3 deletions(-) create mode 100644 lib/util/cache/repository/index.spec.ts create mode 100644 lib/util/cache/repository/index.ts diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md index c5f1568323691c..dae88c9a9e6092 100644 --- a/docs/usage/self-hosted-configuration.md +++ b/docs/usage/self-hosted-configuration.md @@ -149,6 +149,12 @@ If this value is set then Renovate will use Redis for its global cache instead o ## repositories +## repositoryCache + +Set this to `"enabled"` to have Renovate maintain a JSON file cache per-repository to speed up extractions. Set to `"reset"` if you ever need to bypass the cache and have it overwritten. JSON files will be stored inside the `cacheDir` beside the existing file-based package cache. + +Warning: this is an experimental feature and may be modified or removed in a future non-major release. + ## requireConfig ## skipInstalls diff --git a/lib/config/common.ts b/lib/config/common.ts index f212f835574eb1..a5537c876beb6b 100644 --- a/lib/config/common.ts +++ b/lib/config/common.ts @@ -9,6 +9,8 @@ export type RenovateConfigStage = | 'branch' | 'pr'; +export type RepositoryCacheConfig = 'disabled' | 'enabled' | 'reset'; + export interface GroupConfig extends Record { branchName?: string; branchTopic?: string; @@ -45,6 +47,8 @@ export interface RenovateSharedConfig { rebaseLabel?: string; rebaseWhen?: string; recreateClosed?: boolean; + repository?: string; + repositoryCache?: RepositoryCacheConfig; requiredStatusChecks?: string[]; schedule?: string[]; semanticCommits?: boolean; diff --git a/lib/config/definitions.ts b/lib/config/definitions.ts index ca7e6d77a4848e..554148d1f9460c 100644 --- a/lib/config/definitions.ts +++ b/lib/config/definitions.ts @@ -204,6 +204,15 @@ const options: RenovateOptions[] = [ cli: false, env: false, }, + { + name: 'repositoryCache', + description: 'Option to do repository extract caching.', + admin: true, + type: 'string', + allowedValues: ['disabled', 'enabled', 'reset'], + stage: 'repository', + default: 'disabled', + }, { name: 'force', description: diff --git a/lib/util/cache/repository/index.spec.ts b/lib/util/cache/repository/index.spec.ts new file mode 100644 index 00000000000000..b1b5d69dedb19c --- /dev/null +++ b/lib/util/cache/repository/index.spec.ts @@ -0,0 +1,39 @@ +import * as _fs from 'fs-extra'; +import { mocked } from '../../../../test/util'; +import * as repositoryCache from '.'; + +jest.mock('fs-extra'); + +const fs = mocked(_fs); + +describe('lib/util/cache/repository', () => { + const config = { + cacheDir: '/tmp/renovate/cache/', + platform: 'github', + repository: 'abc/def', + }; + it('catches and returns', async () => { + await repositoryCache.initialize({}); + expect(fs.readFile.mock.calls).toHaveLength(0); + }); + it('returns if cache not enabled', async () => { + await repositoryCache.initialize({ + ...config, + repositoryCache: 'disabled', + }); + expect(fs.readFile.mock.calls).toHaveLength(0); + }); + it('reads from cache and finalizes', async () => { + fs.readFile.mockResolvedValueOnce('{}' as any); + await repositoryCache.initialize({ + ...config, + repositoryCache: 'enabled', + }); + await repositoryCache.finalize(); + expect(fs.readFile.mock.calls).toHaveLength(1); + expect(fs.outputFile.mock.calls).toHaveLength(1); + }); + it('gets', () => { + expect(repositoryCache.getCache()).toEqual({}); + }); +}); diff --git a/lib/util/cache/repository/index.ts b/lib/util/cache/repository/index.ts new file mode 100644 index 00000000000000..395316808c4594 --- /dev/null +++ b/lib/util/cache/repository/index.ts @@ -0,0 +1,58 @@ +import * as fs from 'fs-extra'; +import { join } from 'upath'; +import { RenovateConfig, RepositoryCacheConfig } from '../../../config/common'; +import { logger } from '../../../logger'; +import { PackageFile } from '../../../manager/common'; + +export interface BaseBranchCache { + sha: string; // branch commit sha + configHash: string; // object hash of config + packageFiles: PackageFile[]; // extract result +} + +export interface Cache { + init?: { + configFile: string; + contents: RenovateConfig; + }; + scan?: Record; +} + +let repositoryCache: RepositoryCacheConfig = 'disabled'; +let cacheFileName: string; +let cache: Cache = Object.create({}); + +export function getCacheFileName(config: RenovateConfig): string { + return join( + config.cacheDir, + '/renovate/repository/', + config.platform, + config.repository + '.json' + ); +} + +export async function initialize(config: RenovateConfig): Promise { + try { + cacheFileName = getCacheFileName(config); + repositoryCache = config.repositoryCache; + if (repositoryCache !== 'enabled') { + logger.debug('Skipping repository cache'); + cache = {}; + return; + } + cache = JSON.parse(await fs.readFile(cacheFileName, 'utf8')); + logger.debug({ cacheFileName }, 'Read repository cache'); + } catch (err) { + logger.debug({ cacheFileName }, 'No repository cache found'); + } +} + +export function getCache(): Cache { + return cache; +} + +export async function finalize(): Promise { + if (repositoryCache !== 'disabled') { + await fs.outputFile(cacheFileName, JSON.stringify(cache)); + } +} diff --git a/lib/workers/repository/finalise/index.ts b/lib/workers/repository/finalise/index.ts index 1d893607437ac5..c8ff40d7c7755b 100644 --- a/lib/workers/repository/finalise/index.ts +++ b/lib/workers/repository/finalise/index.ts @@ -1,5 +1,6 @@ import { RenovateConfig } from '../../../config'; import { platform } from '../../../platform'; +import * as repositoryCache from '../../../util/cache/repository'; import { pruneStaleBranches } from './prune'; // istanbul ignore next @@ -7,6 +8,7 @@ export async function finaliseRepo( config: RenovateConfig, branchList: string[] ): Promise { + await repositoryCache.finalize(); await pruneStaleBranches(config, branchList); await platform.ensureIssueClosing( `Action Required: Fix Renovate Configuration` diff --git a/lib/workers/repository/init/config.ts b/lib/workers/repository/init/config.ts index cdc7d8fa1fc149..c67276686c57fe 100644 --- a/lib/workers/repository/init/config.ts +++ b/lib/workers/repository/init/config.ts @@ -12,6 +12,8 @@ import * as npmApi from '../../../datasource/npm'; import { logger } from '../../../logger'; import { platform } from '../../../platform'; import { ExternalHostError } from '../../../types/errors/external-host-error'; +import { getCache } from '../../../util/cache/repository'; +import { clone } from '../../../util/clone'; import { readLocalFile } from '../../../util/gitfs'; import * as hostRules from '../../../util/host-rules'; import { flattenPackageRules } from './flatten'; @@ -121,6 +123,11 @@ export async function mergeRenovateConfig( } logger.debug({ configFile, config: renovateJson }, 'Repository config'); } + const cache = getCache(); + cache.init = { + configFile, + contents: clone(renovateJson), + }; const migratedConfig = await migrateAndValidate(config, renovateJson); if (migratedConfig.errors.length) { const error = new Error(CONFIG_VALIDATION); diff --git a/lib/workers/repository/init/index.ts b/lib/workers/repository/init/index.ts index f0c901d5055789..040cd7f0615126 100644 --- a/lib/workers/repository/init/index.ts +++ b/lib/workers/repository/init/index.ts @@ -2,6 +2,7 @@ import { RenovateConfig } from '../../../config'; import { logger } from '../../../logger'; import { platform } from '../../../platform'; import * as memCache from '../../../util/cache/memory'; +import * as repositoryCache from '../../../util/cache/repository'; import { checkIfConfigured } from '../configured'; import { checkOnboardingBranch } from '../onboarding/branch'; import { initApis } from './apis'; @@ -12,6 +13,7 @@ import { detectVulnerabilityAlerts } from './vulnerability'; export async function initRepo(input: RenovateConfig): Promise { memCache.init(); + await repositoryCache.initialize(input); let config: RenovateConfig = { ...input, errors: [], diff --git a/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap b/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap index 20f00a8ca662d9..2b515357e01873 100644 --- a/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap +++ b/lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap @@ -1,6 +1,8 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`workers/repository/process/extract-update extract() runs 1`] = ` +exports[`workers/repository/process/extract-update extract() runs with baseBranches 1`] = `undefined`; + +exports[`workers/repository/process/extract-update extract() runs with no baseBranches 1`] = ` Object { "branchList": Array [ "branchName", diff --git a/lib/workers/repository/process/extract-update.spec.ts b/lib/workers/repository/process/extract-update.spec.ts index 429cd72ea79ec8..aa5ddeed6323d7 100644 --- a/lib/workers/repository/process/extract-update.spec.ts +++ b/lib/workers/repository/process/extract-update.spec.ts @@ -1,4 +1,6 @@ +import hash from 'object-hash'; import { mocked } from '../../../../test/util'; +import * as _repositoryCache from '../../../util/cache/repository'; import * as _branchify from '../updates/branchify'; import { extract, lookup, update } from './extract-update'; @@ -7,8 +9,10 @@ jest.mock('./sort'); jest.mock('./fetch'); jest.mock('../updates/branchify'); jest.mock('../extract'); +jest.mock('../../../util/cache/repository'); const branchify = mocked(_branchify); +const repositoryCache = mocked(_repositoryCache); branchify.branchifyUpgrades.mockResolvedValueOnce({ branches: [{ branchName: 'some-branch', upgrades: [] }], @@ -17,15 +21,46 @@ branchify.branchifyUpgrades.mockResolvedValueOnce({ describe('workers/repository/process/extract-update', () => { describe('extract()', () => { - it('runs', async () => { + it('runs with no baseBranches', async () => { const config = { repoIsOnboarded: true, suppressNotifications: ['deprecationWarningIssues'], }; + repositoryCache.getCache.mockReturnValueOnce({}); const packageFiles = await extract(config); const res = await lookup(config, packageFiles); expect(res).toMatchSnapshot(); await expect(update(config, res.branches)).resolves.not.toThrow(); }); + it('runs with baseBranches', async () => { + const config = { + baseBranches: ['master', 'dev'], + repoIsOnboarded: true, + suppressNotifications: ['deprecationWarningIssues'], + }; + repositoryCache.getCache.mockReturnValueOnce({}); + const packageFiles = await extract(config); + expect(packageFiles).toMatchSnapshot(); + }); + it('uses repository cache', async () => { + const packageFiles = []; + const config = { + repoIsOnboarded: true, + suppressNotifications: ['deprecationWarningIssues'], + baseBranch: 'master', + baseBranchSha: 'abc123', + }; + repositoryCache.getCache.mockReturnValueOnce({ + scan: { + master: { + sha: config.baseBranchSha, + configHash: hash(config).toString(), + packageFiles, + }, + }, + }); + const res = await extract(config); + expect(res).toEqual(packageFiles); + }); }); }); diff --git a/lib/workers/repository/process/extract-update.ts b/lib/workers/repository/process/extract-update.ts index 2c4d9bcba3fd21..e27076a4c34ae8 100644 --- a/lib/workers/repository/process/extract-update.ts +++ b/lib/workers/repository/process/extract-update.ts @@ -1,6 +1,9 @@ +import is from '@sindresorhus/is'; +import hash from 'object-hash'; import { RenovateConfig } from '../../../config'; import { logger } from '../../../logger'; import { PackageFile } from '../../../manager/common'; +import { getCache } from '../../../util/cache/repository'; import { BranchConfig } from '../../common'; import { extractAllDependencies } from '../extract'; import { branchifyUpgrades } from '../updates/branchify'; @@ -47,7 +50,36 @@ export async function extract( config: RenovateConfig ): Promise> { logger.debug('extract()'); - const packageFiles = await extractAllDependencies(config); + const { baseBranch, baseBranchSha } = config; + let packageFiles; + const cache = getCache(); + const cachedExtract = cache?.scan?.[baseBranch]; + const configHash = hash(config); + // istanbul ignore if + if ( + cachedExtract?.sha === baseBranchSha && + cachedExtract?.configHash === configHash + ) { + logger.debug({ baseBranch, baseBranchSha }, 'Found cached extract'); + packageFiles = cachedExtract.packageFiles; + } else { + packageFiles = await extractAllDependencies(config); + cache.scan = cache.scan || Object.create({}); + cache.scan[baseBranch] = { + sha: baseBranchSha, + configHash, + packageFiles, + }; + // Clean up cached branch extracts + const baseBranches = is.nonEmptyArray(config.baseBranches) + ? config.baseBranches + : [baseBranch]; + Object.keys(cache.scan).forEach((branchName) => { + if (!baseBranches.includes(branchName)) { + delete cache.scan[branchName]; + } + }); + } const stats = extractStats(packageFiles); logger.info( { baseBranch: config.baseBranch, stats }, diff --git a/package.json b/package.json index 6532a7305773bf..f5b5da61f20dcf 100644 --- a/package.json +++ b/package.json @@ -151,6 +151,7 @@ "moment-timezone": "0.5.31", "node-emoji": "1.10.0", "node-html-parser": "1.2.20", + "object-hash": "2.0.3", "p-all": "2.1.0", "p-map": "4.0.0", "parse-diff": "0.7.0", diff --git a/yarn.lock b/yarn.lock index 654696d73335bd..1d76acaad5a4a6 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7474,6 +7474,11 @@ object-copy@^0.1.0: define-property "^0.2.5" kind-of "^3.0.3" +object-hash@2.0.3: + version "2.0.3" + resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea" + integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg== + object-inspect@^1.7.0: version "1.8.0" resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0"