feat: repositoryCache (#6589)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

docs/usage/self-hosted-configuration.md

@@ -149,6 +149,12 @@ If this value is set then Renovate will use Redis for its global cache instead o
 
 ## repositories
 
+## repositoryCache
+
+Set this to `"enabled"` to have Renovate maintain a JSON file cache per-repository to speed up extractions. Set to `"reset"` if you ever need to bypass the cache and have it overwritten. JSON files will be stored inside the `cacheDir` beside the existing file-based package cache.
+
+Warning: this is an experimental feature and may be modified or removed in a future non-major release.
+
 ## requireConfig
 
 ## skipInstalls

lib/config/common.ts

@@ -9,6 +9,8 @@ export type RenovateConfigStage =
   | 'branch'
   | 'pr';
 
+export type RepositoryCacheConfig = 'disabled' | 'enabled' | 'reset';
+
 export interface GroupConfig extends Record<string, unknown> {
   branchName?: string;
   branchTopic?: string;
@@ -45,6 +47,8 @@ export interface RenovateSharedConfig {
   rebaseLabel?: string;
   rebaseWhen?: string;
   recreateClosed?: boolean;
+  repository?: string;
+  repositoryCache?: RepositoryCacheConfig;
   requiredStatusChecks?: string[];
   schedule?: string[];
   semanticCommits?: boolean;

lib/config/definitions.ts

@@ -204,6 +204,15 @@ const options: RenovateOptions[] = [
     cli: false,
     env: false,
   },
+  {
+    name: 'repositoryCache',
+    description: 'Option to do repository extract caching.',
+    admin: true,
+    type: 'string',
+    allowedValues: ['disabled', 'enabled', 'reset'],
+    stage: 'repository',
+    default: 'disabled',
+  },
   {
     name: 'force',
     description:

lib/util/cache/repository/index.spec.ts

@@ -0,0 +1,39 @@
+import * as _fs from 'fs-extra';
+import { mocked } from '../../../../test/util';
+import * as repositoryCache from '.';
+
+jest.mock('fs-extra');
+
+const fs = mocked(_fs);
+
+describe('lib/util/cache/repository', () => {
+  const config = {
+    cacheDir: '/tmp/renovate/cache/',
+    platform: 'github',
+    repository: 'abc/def',
+  };
+  it('catches and returns', async () => {
+    await repositoryCache.initialize({});
+    expect(fs.readFile.mock.calls).toHaveLength(0);
+  });
+  it('returns if cache not enabled', async () => {
+    await repositoryCache.initialize({
+      ...config,
+      repositoryCache: 'disabled',
+    });
+    expect(fs.readFile.mock.calls).toHaveLength(0);
+  });
+  it('reads from cache and finalizes', async () => {
+    fs.readFile.mockResolvedValueOnce('{}' as any);
+    await repositoryCache.initialize({
+      ...config,
+      repositoryCache: 'enabled',
+    });
+    await repositoryCache.finalize();
+    expect(fs.readFile.mock.calls).toHaveLength(1);
+    expect(fs.outputFile.mock.calls).toHaveLength(1);
+  });
+  it('gets', () => {
+    expect(repositoryCache.getCache()).toEqual({});
+  });
+});

lib/util/cache/repository/index.ts

@@ -0,0 +1,58 @@
+import * as fs from 'fs-extra';
+import { join } from 'upath';
+import { RenovateConfig, RepositoryCacheConfig } from '../../../config/common';
+import { logger } from '../../../logger';
+import { PackageFile } from '../../../manager/common';
+
+export interface BaseBranchCache {
+  sha: string; // branch commit sha
+  configHash: string; // object hash of config
+  packageFiles: PackageFile[]; // extract result
+}
+
+export interface Cache {
+  init?: {
+    configFile: string;
+    contents: RenovateConfig;
+  };
+  scan?: Record<string, BaseBranchCache>;
+}
+
+let repositoryCache: RepositoryCacheConfig = 'disabled';
+let cacheFileName: string;
+let cache: Cache = Object.create({});
+
+export function getCacheFileName(config: RenovateConfig): string {
+  return join(
+    config.cacheDir,
+    '/renovate/repository/',
+    config.platform,
+    config.repository + '.json'
+  );
+}
+
+export async function initialize(config: RenovateConfig): Promise<void> {
+  try {
+    cacheFileName = getCacheFileName(config);
+    repositoryCache = config.repositoryCache;
+    if (repositoryCache !== 'enabled') {
+      logger.debug('Skipping repository cache');
+      cache = {};
+      return;
+    }
+    cache = JSON.parse(await fs.readFile(cacheFileName, 'utf8'));
+    logger.debug({ cacheFileName }, 'Read repository cache');
+  } catch (err) {
+    logger.debug({ cacheFileName }, 'No repository cache found');
+  }
+}
+
+export function getCache(): Cache {
+  return cache;
+}
+
+export async function finalize(): Promise<void> {
+  if (repositoryCache !== 'disabled') {
+    await fs.outputFile(cacheFileName, JSON.stringify(cache));
+  }
+}

lib/workers/repository/finalise/index.ts

@@ -1,12 +1,14 @@
 import { RenovateConfig } from '../../../config';
 import { platform } from '../../../platform';
+import * as repositoryCache from '../../../util/cache/repository';
 import { pruneStaleBranches } from './prune';
 
 // istanbul ignore next
 export async function finaliseRepo(
   config: RenovateConfig,
   branchList: string[]
 ): Promise<void> {
+  await repositoryCache.finalize();
   await pruneStaleBranches(config, branchList);
   await platform.ensureIssueClosing(
     `Action Required: Fix Renovate Configuration`

lib/workers/repository/init/config.ts

@@ -12,6 +12,8 @@ import * as npmApi from '../../../datasource/npm';
 import { logger } from '../../../logger';
 import { platform } from '../../../platform';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
+import { getCache } from '../../../util/cache/repository';
+import { clone } from '../../../util/clone';
 import { readLocalFile } from '../../../util/gitfs';
 import * as hostRules from '../../../util/host-rules';
 import { flattenPackageRules } from './flatten';
@@ -121,6 +123,11 @@ export async function mergeRenovateConfig(
     }
     logger.debug({ configFile, config: renovateJson }, 'Repository config');
   }
+  const cache = getCache();
+  cache.init = {
+    configFile,
+    contents: clone(renovateJson),
+  };
   const migratedConfig = await migrateAndValidate(config, renovateJson);
   if (migratedConfig.errors.length) {
     const error = new Error(CONFIG_VALIDATION);

lib/workers/repository/init/index.ts

@@ -2,6 +2,7 @@ import { RenovateConfig } from '../../../config';
 import { logger } from '../../../logger';
 import { platform } from '../../../platform';
 import * as memCache from '../../../util/cache/memory';
+import * as repositoryCache from '../../../util/cache/repository';
 import { checkIfConfigured } from '../configured';
 import { checkOnboardingBranch } from '../onboarding/branch';
 import { initApis } from './apis';
@@ -12,6 +13,7 @@ import { detectVulnerabilityAlerts } from './vulnerability';
 
 export async function initRepo(input: RenovateConfig): Promise<RenovateConfig> {
   memCache.init();
+  await repositoryCache.initialize(input);
   let config: RenovateConfig = {
     ...input,
     errors: [],

lib/workers/repository/process/__snapshots__/extract-update.spec.ts.snap

@@ -1,6 +1,8 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
-exports[`workers/repository/process/extract-update extract() runs 1`] = `
+exports[`workers/repository/process/extract-update extract() runs with baseBranches 1`] = `undefined`;
+
+exports[`workers/repository/process/extract-update extract() runs with no baseBranches 1`] = `
 Object {
   "branchList": Array [
     "branchName",

lib/workers/repository/process/extract-update.spec.ts

@@ -1,4 +1,6 @@
+import hash from 'object-hash';
 import { mocked } from '../../../../test/util';
+import * as _repositoryCache from '../../../util/cache/repository';
 import * as _branchify from '../updates/branchify';
 import { extract, lookup, update } from './extract-update';
 
@@ -7,8 +9,10 @@ jest.mock('./sort');
 jest.mock('./fetch');
 jest.mock('../updates/branchify');
 jest.mock('../extract');
+jest.mock('../../../util/cache/repository');
 
 const branchify = mocked(_branchify);
+const repositoryCache = mocked(_repositoryCache);
 
 branchify.branchifyUpgrades.mockResolvedValueOnce({
   branches: [{ branchName: 'some-branch', upgrades: [] }],
@@ -17,15 +21,46 @@ branchify.branchifyUpgrades.mockResolvedValueOnce({
 
 describe('workers/repository/process/extract-update', () => {
   describe('extract()', () => {
-    it('runs', async () => {
+    it('runs with no baseBranches', async () => {
       const config = {
         repoIsOnboarded: true,
         suppressNotifications: ['deprecationWarningIssues'],
       };
+      repositoryCache.getCache.mockReturnValueOnce({});
       const packageFiles = await extract(config);
       const res = await lookup(config, packageFiles);
       expect(res).toMatchSnapshot();
       await expect(update(config, res.branches)).resolves.not.toThrow();
     });
+    it('runs with baseBranches', async () => {
+      const config = {
+        baseBranches: ['master', 'dev'],
+        repoIsOnboarded: true,
+        suppressNotifications: ['deprecationWarningIssues'],
+      };
+      repositoryCache.getCache.mockReturnValueOnce({});
+      const packageFiles = await extract(config);
+      expect(packageFiles).toMatchSnapshot();
+    });
+    it('uses repository cache', async () => {
+      const packageFiles = [];
+      const config = {
+        repoIsOnboarded: true,
+        suppressNotifications: ['deprecationWarningIssues'],
+        baseBranch: 'master',
+        baseBranchSha: 'abc123',
+      };
+      repositoryCache.getCache.mockReturnValueOnce({
+        scan: {
+          master: {
+            sha: config.baseBranchSha,
+            configHash: hash(config).toString(),
+            packageFiles,
+          },
+        },
+      });
+      const res = await extract(config);
+      expect(res).toEqual(packageFiles);
+    });
   });
 });

lib/workers/repository/process/extract-update.ts

@@ -1,6 +1,9 @@
+import is from '@sindresorhus/is';
+import hash from 'object-hash';
 import { RenovateConfig } from '../../../config';
 import { logger } from '../../../logger';
 import { PackageFile } from '../../../manager/common';
+import { getCache } from '../../../util/cache/repository';
 import { BranchConfig } from '../../common';
 import { extractAllDependencies } from '../extract';
 import { branchifyUpgrades } from '../updates/branchify';
@@ -47,7 +50,36 @@ export async function extract(
   config: RenovateConfig
 ): Promise<Record<string, PackageFile[]>> {
   logger.debug('extract()');
-  const packageFiles = await extractAllDependencies(config);
+  const { baseBranch, baseBranchSha } = config;
+  let packageFiles;
+  const cache = getCache();
+  const cachedExtract = cache?.scan?.[baseBranch];
+  const configHash = hash(config);
+  // istanbul ignore if
+  if (
+    cachedExtract?.sha === baseBranchSha &&
+    cachedExtract?.configHash === configHash
+  ) {
+    logger.debug({ baseBranch, baseBranchSha }, 'Found cached extract');
+    packageFiles = cachedExtract.packageFiles;
+  } else {
+    packageFiles = await extractAllDependencies(config);
+    cache.scan = cache.scan || Object.create({});
+    cache.scan[baseBranch] = {
+      sha: baseBranchSha,
+      configHash,
+      packageFiles,
+    };
+    // Clean up cached branch extracts
+    const baseBranches = is.nonEmptyArray(config.baseBranches)
+      ? config.baseBranches
+      : [baseBranch];
+    Object.keys(cache.scan).forEach((branchName) => {
+      if (!baseBranches.includes(branchName)) {
+        delete cache.scan[branchName];
+      }
+    });
+  }
   const stats = extractStats(packageFiles);
   logger.info(
     { baseBranch: config.baseBranch, stats },

package.json

@@ -151,6 +151,7 @@
     "moment-timezone": "0.5.31",
     "node-emoji": "1.10.0",
     "node-html-parser": "1.2.20",
+    "object-hash": "2.0.3",
     "p-all": "2.1.0",
     "p-map": "4.0.0",
     "parse-diff": "0.7.0",

yarn.lock

@@ -7474,6 +7474,11 @@ object-copy@^0.1.0:
     define-property "^0.2.5"
     kind-of "^3.0.3"
 
+object-hash@2.0.3:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/object-hash/-/object-hash-2.0.3.tgz#d12db044e03cd2ca3d77c0570d87225b02e1e6ea"
+  integrity sha512-JPKn0GMu+Fa3zt3Bmr66JhokJU5BaNBIh4ZeTlaCBzrBsOeXzwcKKAK1tbLiPKgvwmPXsDvvLHoWh5Bm7ofIYg==
+
 object-inspect@^1.7.0:
   version "1.8.0"
   resolved "https://registry.yarnpkg.com/object-inspect/-/object-inspect-1.8.0.tgz#df807e5ecf53a609cc6bfe93eac3cc7be5b3a9d0"