feat(cache): Compression for repository cache (#15289)

lib/util/cache/repository/common.ts

@@ -2,30 +2,46 @@ import is from '@sindresorhus/is';
 import type { RepoCacheData, RepoCacheRecord } from './types';
 
 // Increment this whenever there could be incompatibilities between old and new cache structure
-export const CACHE_REVISION = 11;
+export const CACHE_REVISION = 12;
 
-export function isValidCacheRecord(
+export function isValidRev10(
   input: unknown,
   repo?: string
-): input is RepoCacheRecord {
+): input is RepoCacheData & { repository?: string; revision?: number } {
   return (
     is.plainObject(input) &&
-    is.string(input.repository) &&
     is.safeInteger(input.revision) &&
-    (!repo || repo === input.repository) &&
-    input.revision === CACHE_REVISION
+    input.revision === 10 &&
+    is.string(input.repository) &&
+    (!repo || repo === input.repository)
   );
 }
 
-export function canBeMigratedToV11(
+export function isValidRev11(
   input: unknown,
   repo?: string
-): input is RepoCacheData & { repository?: string; revision?: number } {
+): input is { repository: string; revision: number; data: RepoCacheData } {
   return (
     is.plainObject(input) &&
+    is.safeInteger(input.revision) &&
+    input.revision === 11 &&
     is.string(input.repository) &&
+    is.plainObject(input.data) &&
+    (!repo || repo === input.repository)
+  );
+}
+
+export function isValidRev12(
+  input: unknown,
+  repo?: string
+): input is RepoCacheRecord {
+  return (
+    is.plainObject(input) &&
     is.safeInteger(input.revision) &&
+    input.revision === CACHE_REVISION &&
+    is.string(input.repository) &&
     (!repo || repo === input.repository) &&
-    input.revision === 10
+    is.string(input.payload) &&
+    is.string(input.hash)
   );
 }

lib/util/cache/repository/impl/local.spec.ts

@@ -1,3 +1,6 @@
+import { promisify } from 'util';
+import zlib from 'zlib';
+import hasha from 'hasha';
 import { fs } from '../../../../../test/util';
 import { GlobalConfig } from '../../../../config/global';
 import { CACHE_REVISION } from '../common';
@@ -6,6 +9,21 @@ import { LocalRepoCache } from './local';
 
 jest.mock('../../../fs');
 
+const compress = promisify(zlib.brotliCompress);
+
+async function createCacheRecord(
+  data: RepoCacheData,
+  repository = 'some/repo'
+): Promise<RepoCacheRecord> {
+  const revision = CACHE_REVISION;
+  const jsonStr = JSON.stringify(data);
+  const hash = hasha(jsonStr, { algorithm: 'sha256' });
+  const compressed = await compress(jsonStr);
+  const payload = compressed.toString('base64');
+  const record: RepoCacheRecord = { revision, repository, payload, hash };
+  return record;
+}
+
 describe('util/cache/repository/impl/local', () => {
   beforeEach(() => {
     GlobalConfig.set({ cacheDir: '/tmp/cache' });
@@ -16,22 +34,18 @@ describe('util/cache/repository/impl/local', () => {
     expect(localRepoCache.getData()).toBeEmpty();
   });
 
-  it('loads valid cache from disk', async () => {
+  it('loads previously stored cache from disk', async () => {
     const data: RepoCacheData = { semanticCommits: 'enabled' };
-    const cache: RepoCacheRecord = {
-      repository: 'some/repo',
-      revision: CACHE_REVISION,
-      data,
-    };
-    fs.readFile.mockResolvedValue(JSON.stringify(cache));
+    const cacheRecord = await createCacheRecord(data);
+    fs.readFile.mockResolvedValue(JSON.stringify(cacheRecord));
     const localRepoCache = new LocalRepoCache('github', 'some/repo');
 
     await localRepoCache.load();
 
     expect(localRepoCache.getData()).toEqual(data);
   });
 
-  it('migrates revision from 10 to 11', async () => {
+  it('migrates revision from 10 to 12', async () => {
     fs.readFile.mockResolvedValue(
       JSON.stringify({
         revision: 10,
@@ -40,18 +54,35 @@ describe('util/cache/repository/impl/local', () => {
       })
     );
     const localRepoCache = new LocalRepoCache('github', 'some/repo');
-    await localRepoCache.load();
 
+    await localRepoCache.load();
     await localRepoCache.save();
 
+    const cacheRecord = await createCacheRecord({ semanticCommits: 'enabled' });
     expect(fs.outputFile).toHaveBeenCalledWith(
       '/tmp/cache/renovate/repository/github/some/repo.json',
+      JSON.stringify(cacheRecord)
+    );
+  });
+
+  it('migrates revision from 11 to 12', async () => {
+    fs.readFile.mockResolvedValue(
       JSON.stringify({
-        revision: CACHE_REVISION,
+        revision: 11,
         repository: 'some/repo',
         data: { semanticCommits: 'enabled' },
       })
     );
+    const localRepoCache = new LocalRepoCache('github', 'some/repo');
+
+    await localRepoCache.load();
+    await localRepoCache.save();
+
+    const cacheRecord = await createCacheRecord({ semanticCommits: 'enabled' });
+    expect(fs.outputFile).toHaveBeenCalledWith(
+      '/tmp/cache/renovate/repository/github/some/repo.json',
+      JSON.stringify(cacheRecord)
+    );
   });
 
   it('does not migrate from older revisions to 11', async () => {
@@ -89,42 +120,31 @@ describe('util/cache/repository/impl/local', () => {
   });
 
   it('resets if repository does not match', async () => {
-    fs.readFile.mockResolvedValueOnce(
-      JSON.stringify({
-        revision: CACHE_REVISION,
-        repository: 'foo/bar',
-        data: { semanticCommits: 'enabled' },
-      }) as never
-    );
+    const cacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
+    fs.readFile.mockResolvedValueOnce(JSON.stringify(cacheRecord) as never);
 
     const localRepoCache = new LocalRepoCache('github', 'some/repo');
     await localRepoCache.load();
 
     expect(localRepoCache.getData()).toEqual({});
   });
 
-  it('saves cache data to file', async () => {
-    fs.readFile.mockResolvedValueOnce(
-      JSON.stringify({
-        revision: CACHE_REVISION,
-        repository: 'some/repo',
-        data: { semanticCommits: 'enabled' },
-      })
-    );
+  it('saves modified cache data to file', async () => {
+    const oldCacheRecord = createCacheRecord({ semanticCommits: 'enabled' });
+    fs.readFile.mockResolvedValueOnce(JSON.stringify(oldCacheRecord));
     const localRepoCache = new LocalRepoCache('github', 'some/repo');
-    await localRepoCache.load();
 
+    await localRepoCache.load();
     const data = localRepoCache.getData();
     data.semanticCommits = 'disabled';
     await localRepoCache.save();
 
+    const newCacheRecord = await createCacheRecord({
+      semanticCommits: 'disabled',
+    });
     expect(fs.outputFile).toHaveBeenCalledWith(
       '/tmp/cache/renovate/repository/github/some/repo.json',
-      JSON.stringify({
-        revision: CACHE_REVISION,
-        repository: 'some/repo',
-        data: { semanticCommits: 'disabled' },
-      })
+      JSON.stringify(newCacheRecord)
     );
   });
 });

lib/util/cache/repository/impl/local.ts

@@ -1,21 +1,30 @@
+import { promisify } from 'util';
+import zlib from 'zlib';
+import hasha from 'hasha';
 import upath from 'upath';
 import { GlobalConfig } from '../../../../config/global';
 import { logger } from '../../../../logger';
 import { outputFile, readFile } from '../../../fs';
 import {
   CACHE_REVISION,
-  canBeMigratedToV11,
-  isValidCacheRecord,
+  isValidRev10,
+  isValidRev11,
+  isValidRev12,
 } from '../common';
 import type { RepoCacheRecord } from '../types';
 import { RepoCacheBase } from './base';
 
+const compress = promisify(zlib.brotliCompress);
+const decompress = promisify(zlib.brotliDecompress);
+
 export class LocalRepoCache extends RepoCacheBase {
+  private oldHash: string | null = null;
+
   constructor(private platform: string, private repository: string) {
     super();
   }
 
-  private getCacheFileName(): string {
+  public getCacheFileName(): string {
     const cacheDir = GlobalConfig.get('cacheDir');
     const repoCachePath = '/renovate/repository/';
     const platform = this.platform;
@@ -29,17 +38,32 @@ export class LocalRepoCache extends RepoCacheBase {
       const cacheFileName = this.getCacheFileName();
       const rawCache = await readFile(cacheFileName, 'utf8');
       const oldCache = JSON.parse(rawCache);
-      if (isValidCacheRecord(oldCache, this.repository)) {
-        this.data = oldCache.data;
+
+      if (isValidRev12(oldCache, this.repository)) {
+        const compressed = Buffer.from(oldCache.payload, 'base64');
+        const uncompressed = await decompress(compressed);
+        const jsonStr = uncompressed.toString('utf8');
+        this.data = JSON.parse(jsonStr);
+        this.oldHash = oldCache.hash;
         logger.debug('Repository cache is valid');
-      } else if (canBeMigratedToV11(oldCache, this.repository)) {
+        return;
+      }
+
+      if (isValidRev11(oldCache, this.repository)) {
+        this.data = oldCache.data;
+        logger.debug('Repository cache is migrated from 11 revision');
+        return;
+      }
+
+      if (isValidRev10(oldCache, this.repository)) {
         delete oldCache.repository;
         delete oldCache.revision;
         this.data = oldCache;
-        logger.debug('Repository cache is migrated');
-      } else {
-        logger.debug('Repository cache is invalid');
+        logger.debug('Repository cache is migrated from 10 revision');
+        return;
       }
+
+      logger.debug('Repository cache is invalid');
     } catch (err) {
       logger.debug({ cacheFileName }, 'Repository cache not found');
     }
@@ -50,7 +74,13 @@ export class LocalRepoCache extends RepoCacheBase {
     const revision = CACHE_REVISION;
     const repository = this.repository;
     const data = this.getData();
-    const record: RepoCacheRecord = { revision, repository, data };
-    await outputFile(cacheFileName, JSON.stringify(record));
+    const jsonStr = JSON.stringify(data);
+    const hash = await hasha.async(jsonStr, { algorithm: 'sha256' });
+    if (hash !== this.oldHash) {
+      const compressed = await compress(jsonStr);
+      const payload = compressed.toString('base64');
+      const record: RepoCacheRecord = { revision, repository, payload, hash };
+      await outputFile(cacheFileName, JSON.stringify(record));
+    }
   }
 }

lib/util/cache/repository/types.ts

@@ -49,7 +49,8 @@ export interface RepoCacheData {
 export interface RepoCacheRecord {
   repository: string;
   revision: number;
-  data: RepoCacheData;
+  payload: string;
+  hash: string;
 }
 
 export interface RepoCache {