refactor(cache): Separate data and metadata for repo cache (#15117)

lib/modules/platform/comment.spec.ts

@@ -1,6 +1,6 @@
 import { mocked, platform } from '../../../test/util';
 import * as _cache from '../../util/cache/repository';
-import type { Cache } from '../../util/cache/repository/types';
+import type { RepoCacheData } from '../../util/cache/repository/types';
 import { ensureComment, ensureCommentRemoval } from './comment';
 
 jest.mock('.');
@@ -9,7 +9,7 @@ jest.mock('../../util/cache/repository');
 const cache = mocked(_cache);
 
 describe('modules/platform/comment', () => {
-  let repoCache: Cache = {};
+  let repoCache: RepoCacheData = {};
 
   beforeEach(() => {
     repoCache = {};

lib/util/cache/repository/index.spec.ts

@@ -1,6 +1,8 @@
 import * as _fs from 'fs-extra';
 import { mocked } from '../../../../test/util';
 import { GlobalConfig } from '../../../config/global';
+import type { RenovateConfig } from '../../../config/types';
+import type { RepoCache } from './types';
 import * as repositoryCache from '.';
 
 jest.mock('fs-extra');
@@ -9,54 +11,121 @@ const fs = mocked(_fs);
 
 describe('util/cache/repository/index', () => {
   beforeEach(() => {
+    repositoryCache.reset();
     jest.resetAllMocks();
     GlobalConfig.set({ cacheDir: '/tmp/renovate/cache/' });
   });
 
-  const config = {
+  const config: RenovateConfig = {
     platform: 'github',
     repository: 'abc/def',
+    repositoryCache: 'enabled',
   };
 
-  it('catches and returns', async () => {
-    await repositoryCache.initialize({});
-    expect(fs.readFile.mock.calls).toHaveLength(0);
-  });
+  const repoCache: RepoCache = {
+    revision: 11,
+    repository: 'abc/def',
+    data: {},
+  };
 
   it('returns if cache not enabled', async () => {
     await repositoryCache.initialize({
       ...config,
       repositoryCache: 'disabled',
     });
-    expect(fs.readFile.mock.calls).toHaveLength(0);
+    expect(fs.readFile).not.toHaveBeenCalled();
   });
 
-  it('resets if invalid', async () => {
-    fs.readFile.mockResolvedValueOnce('{}' as any);
-    await repositoryCache.initialize({
-      ...config,
-      repositoryCache: 'enabled',
-    });
-    expect(repositoryCache.getCache()).toEqual({
-      repository: 'abc/def',
-      revision: repositoryCache.CACHE_REVISION,
-    });
+  it('resets if repository does not match', async () => {
+    fs.readFile.mockResolvedValueOnce(
+      JSON.stringify({
+        ...repoCache,
+        repository: 'foo/bar',
+        data: { semanticCommits: 'enabled' },
+      }) as never
+    );
+
+    await repositoryCache.initialize(config);
+
+    expect(repositoryCache.getCache()).toEqual({});
   });
 
   it('reads from cache and finalizes', async () => {
     fs.readFile.mockResolvedValueOnce(
-      `{"repository":"abc/def","revision":${repositoryCache.CACHE_REVISION}}` as any
+      JSON.stringify({
+        ...repoCache,
+        data: { semanticCommits: 'enabled' },
+      }) as never
     );
-    await repositoryCache.initialize({
-      ...config,
-      repositoryCache: 'enabled',
-    });
+
+    await repositoryCache.initialize(config);
+
+    expect(fs.readFile).toHaveBeenCalled();
+
+    const cache = repositoryCache.getCache();
+    expect(cache).toEqual({ semanticCommits: 'enabled' });
+
+    cache.semanticCommits = 'disabled';
     await repositoryCache.finalize();
-    expect(fs.readFile.mock.calls).toHaveLength(1);
-    expect(fs.outputFile.mock.calls).toHaveLength(1);
+    expect(fs.outputFile).toHaveBeenCalledWith(
+      '/tmp/renovate/cache/renovate/repository/github/abc/def.json',
+      JSON.stringify({
+        revision: 11,
+        repository: 'abc/def',
+        data: { semanticCommits: 'disabled' },
+      })
+    );
+  });
+
+  it('migrates from 10 to 11 revision', async () => {
+    fs.readFile.mockResolvedValueOnce(
+      JSON.stringify({
+        revision: 10,
+        repository: 'abc/def',
+        semanticCommits: 'enabled',
+      }) as never
+    );
+
+    await repositoryCache.initialize(config);
+
+    const cache = repositoryCache.getCache();
+    expect(cache).toEqual({ semanticCommits: 'enabled' });
+
+    cache.semanticCommits = 'disabled';
+    await repositoryCache.finalize();
+    expect(fs.outputFile).toHaveBeenCalledWith(
+      '/tmp/renovate/cache/renovate/repository/github/abc/def.json',
+      JSON.stringify({
+        revision: 11,
+        repository: 'abc/def',
+        data: { semanticCommits: 'disabled' },
+      })
+    );
   });
 
-  it('gets', () => {
+  it('does not migrate from older revisions to 11', async () => {
+    fs.readFile.mockResolvedValueOnce(
+      JSON.stringify({
+        revision: 9,
+        repository: 'abc/def',
+        semanticCommits: 'enabled',
+      }) as never
+    );
+
+    await repositoryCache.initialize(config);
+
+    const cache = repositoryCache.getCache();
+    expect(cache).toEqual({});
+  });
+
+  it('returns empty cache for non-initialized cache', () => {
     expect(repositoryCache.getCache()).toEqual({});
   });
+
+  it('returns empty cache after initialization error', async () => {
+    fs.readFile.mockRejectedValueOnce(new Error('unknown error'));
+    await repositoryCache.initialize(config);
+    const cache = repositoryCache.getCache();
+    expect(cache).toEqual({});
+  });
 });

lib/util/cache/repository/index.ts

@@ -1,3 +1,4 @@
+import is from '@sindresorhus/is';
 import fs from 'fs-extra';
 import upath from 'upath';
 import { GlobalConfig } from '../../../config/global';
@@ -6,70 +7,102 @@ import type {
   RepositoryCacheConfig,
 } from '../../../config/types';
 import { logger } from '../../../logger';
-import type { Cache } from './types';
+import type { RepoCache, RepoCacheData } from './types';
 
 // Increment this whenever there could be incompatibilities between old and new cache structure
-export const CACHE_REVISION = 10;
+const CACHE_REVISION = 11;
 
 let repositoryCache: RepositoryCacheConfig | undefined = 'disabled';
 let cacheFileName: string | null = null;
-let cache: Cache | null = Object.create({});
+
+let repository: string | null | undefined = null;
+let data: RepoCacheData | null = null;
+
+export function reset(): void {
+  repository = null;
+  data = null;
+}
 
 export function getCacheFileName(config: RenovateConfig): string {
-  return upath.join(
-    GlobalConfig.get('cacheDir'),
-    '/renovate/repository/',
-    config.platform,
-    `${config.repository}.json`
-  );
+  const cacheDir = GlobalConfig.get('cacheDir');
+  const repoCachePath = '/renovate/repository/';
+  const platform = config.platform;
+  const fileName = `${config.repository}.json`;
+  return upath.join(cacheDir, repoCachePath, platform, fileName);
 }
 
-function validate(config: RenovateConfig, input: any): Cache | null {
-  if (
-    input &&
+function isCacheValid(
+  config: RenovateConfig,
+  input: unknown
+): input is RepoCache {
+  return (
+    is.plainObject(input) &&
+    is.string(input.repository) &&
+    is.safeInteger(input.revision) &&
     input.repository === config.repository &&
     input.revision === CACHE_REVISION
-  ) {
-    logger.debug('Repository cache is valid');
-    return input as Cache;
-  }
-  logger.info('Repository cache invalidated');
-  // reset
-  return null;
+  );
 }
 
-function createCache(repository?: string): Cache {
-  const res: Cache = Object.create({});
-  res.repository = repository;
-  res.revision = CACHE_REVISION;
-  return res;
+function canBeMigratedToV11(
+  config: RenovateConfig,
+  input: unknown
+): input is RepoCacheData & { repository?: string; revision?: number } {
+  return (
+    is.plainObject(input) &&
+    is.string(input.repository) &&
+    is.safeInteger(input.revision) &&
+    input.repository === config.repository &&
+    input.revision === 10
+  );
 }
 
 export async function initialize(config: RenovateConfig): Promise<void> {
-  cache = null;
+  reset();
+
   try {
     cacheFileName = getCacheFileName(config);
     repositoryCache = config.repositoryCache;
     if (repositoryCache === 'enabled') {
-      cache = validate(
-        config,
-        JSON.parse(await fs.readFile(cacheFileName, 'utf8'))
-      );
+      const rawCache = await fs.readFile(cacheFileName, 'utf8');
+      const oldCache = JSON.parse(rawCache);
+      if (isCacheValid(config, oldCache)) {
+        data = oldCache.data;
+        logger.debug('Repository cache is valid');
+      } else if (canBeMigratedToV11(config, oldCache)) {
+        delete oldCache.repository;
+        delete oldCache.revision;
+        data = oldCache;
+        logger.debug('Repository cache is migrated');
+      } else {
+        logger.debug('Repository cache is invalid');
+      }
     }
   } catch (err) {
     logger.debug({ cacheFileName }, 'Repository cache not found');
   }
-  cache ||= createCache(config.repository);
+
+  repository = config.repository;
+  data ??= {};
 }
 
-export function getCache(): Cache {
-  return cache ?? createCache();
+export function getCache(): RepoCacheData {
+  data ??= {};
+  return data;
 }
 
 export async function finalize(): Promise<void> {
-  if (cacheFileName && cache && repositoryCache !== 'disabled') {
-    await fs.outputFile(cacheFileName, JSON.stringify(cache));
+  if (cacheFileName && repository && data && repositoryCache !== 'disabled') {
+    await fs.outputFile(
+      cacheFileName,
+      JSON.stringify({
+        revision: CACHE_REVISION,
+        repository,
+        data,
+      })
+    );
   }
   cacheFileName = null;
-  cache = Object.create({});
+
+  reset();
 }

lib/util/cache/repository/types.ts

@@ -32,12 +32,10 @@ export interface BranchCache {
   upgrades: BranchUpgradeCache[];
 }
 
-export interface Cache {
+export interface RepoCacheData {
   configFileName?: string;
   semanticCommits?: 'enabled' | 'disabled';
   branches?: BranchCache[];
-  repository?: string;
-  revision?: number;
   init?: RepoInitConfig;
   scan?: Record<string, BaseBranchCache>;
   lastPlatformAutomergeFailure?: string;
@@ -47,3 +45,9 @@ export interface Cache {
   gitConflicts?: GitConflictsCache;
   prComments?: Record<number, Record<string, string>>;
 }
+
+export interface RepoCache {
+  repository: string;
+  revision: number;
+  data: RepoCacheData;
+}

lib/util/git/conflicts-cache.spec.ts

@@ -1,6 +1,6 @@
 import { mocked } from '../../../test/util';
 import * as _repositoryCache from '../cache/repository';
-import type { Cache } from '../cache/repository/types';
+import type { RepoCacheData } from '../cache/repository/types';
 import {
   getCachedConflictResult,
   setCachedConflictResult,
@@ -10,7 +10,7 @@ jest.mock('../cache/repository');
 const repositoryCache = mocked(_repositoryCache);
 
 describe('util/git/conflicts-cache', () => {
-  let repoCache: Cache = {};
+  let repoCache: RepoCacheData = {};
 
   beforeEach(() => {
     repoCache = {};

lib/util/http/github.spec.ts

@@ -10,7 +10,7 @@ import {
 } from '../../constants/error-messages';
 import { GithubReleasesDatasource } from '../../modules/datasource/github-releases';
 import * as _repositoryCache from '../cache/repository';
-import type { Cache } from '../cache/repository/types';
+import type { RepoCacheData } from '../cache/repository/types';
 import * as hostRules from '../host-rules';
 import { GithubHttp, setBaseUrl } from './github';
 
@@ -47,7 +47,7 @@ query(
 
 describe('util/http/github', () => {
   let githubApi: GithubHttp;
-  let repoCache: Cache = {};
+  let repoCache: RepoCacheData = {};
 
   beforeEach(() => {
     githubApi = new GithubHttp();