feat(cache): RENOVATE_X_REPO_CACHE_FORCE_LOCAL (#28291)

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

Author: PhilipAbed

docs/usage/self-hosted-experimental.md

@@ -160,6 +160,10 @@ If set, Renovate will rewrite GitHub Enterprise Server's pagination responses to
 !!! note
     For the GitHub Enterprise Server platform only.
 
+## `RENOVATE_X_REPO_CACHE_FORCE_LOCAL`
+
+If set, Renovate will persist repository cache locally after uploading to S3.
+
 ## `RENOVATE_X_S3_ENDPOINT`
 
 If set, Renovate will use this string as the `endpoint` when instantiating the AWS S3 client.

lib/util/cache/repository/common.ts

@@ -1,2 +1,13 @@
 // Increment this whenever there could be incompatibilities between old and new cache structure
+import upath from 'upath';
+
 export const CACHE_REVISION = 13;
+
+export function getLocalCacheFileName(
+  platform: string,
+  repository: string,
+): string {
+  const repoCachePath = 'renovate/repository/';
+  const fileName = `${repository}.json`;
+  return upath.join(repoCachePath, platform, fileName);
+}

lib/util/cache/repository/impl/local.ts

@@ -1,6 +1,6 @@
-import upath from 'upath';
 import { logger } from '../../../../logger';
 import { cachePathExists, outputCacheFile, readCacheFile } from '../../../fs';
+import { getLocalCacheFileName } from '../common';
 import type { RepoCacheRecord } from '../schema';
 import { RepoCacheBase } from './base';
 
@@ -29,9 +29,6 @@ export class RepoCacheLocal extends RepoCacheBase {
   }
 
   private getCacheFileName(): string {
-    const repoCachePath = 'renovate/repository/';
-    const platform = this.platform;
-    const fileName = `${this.repository}.json`;
-    return upath.join(repoCachePath, platform, fileName);
+    return getLocalCacheFileName(this.platform, this.repository);
   }
 }

lib/util/cache/repository/impl/s3.spec.ts

@@ -8,14 +8,16 @@ import {
   S3Client,
 } from '@aws-sdk/client-s3';
 import { mockClient } from 'aws-sdk-client-mock';
-import { partial } from '../../../../../test/util';
+import { fs, partial } from '../../../../../test/util';
 import { GlobalConfig } from '../../../../config/global';
 import { logger } from '../../../../logger';
 import { parseS3Url } from '../../../s3';
 import type { RepoCacheRecord } from '../schema';
 import { CacheFactory } from './cache-factory';
 import { RepoCacheS3 } from './s3';
 
+jest.mock('../../../fs');
+
 function createGetObjectCommandInput(
   repository: string,
   url: string,
@@ -57,7 +59,7 @@ describe('util/cache/repository/impl/s3', () => {
   let s3Cache: RepoCacheS3;
 
   beforeEach(() => {
-    GlobalConfig.set({ platform: 'github' });
+    GlobalConfig.set({ cacheDir: '/tmp/cache', platform: 'github' });
     s3Mock.reset();
     s3Cache = new RepoCacheS3(repository, '0123456789abcdef', url);
     getObjectCommandInput = createGetObjectCommandInput(repository, url);
@@ -196,4 +198,19 @@ describe('util/cache/repository/impl/s3', () => {
     const cache = CacheFactory.get(repository, '0123456789abcdef', url);
     expect(cache instanceof RepoCacheS3).toBeTrue();
   });
+
+  it('should persists data locally after uploading to s3', async () => {
+    process.env.RENOVATE_X_REPO_CACHE_FORCE_LOCAL = 'true';
+    const putObjectCommandOutput: PutObjectCommandOutput = {
+      $metadata: { attempts: 1, httpStatusCode: 200, totalRetryDelay: 0 },
+    };
+    s3Mock
+      .on(PutObjectCommand, putObjectCommandInput)
+      .resolvesOnce(putObjectCommandOutput);
+    await s3Cache.write(repoCache);
+    expect(fs.outputCacheFile).toHaveBeenCalledWith(
+      'renovate/repository/github/org/repo.json',
+      JSON.stringify(repoCache),
+    );
+  });
 });

lib/util/cache/repository/impl/s3.ts

@@ -5,9 +5,12 @@ import {
   PutObjectCommand,
   PutObjectCommandInput,
 } from '@aws-sdk/client-s3';
+import is from '@sindresorhus/is';
 import { logger } from '../../../../logger';
+import { outputCacheFile } from '../../../fs';
 import { getS3Client, parseS3Url } from '../../../s3';
 import { streamToString } from '../../../streams';
+import { getLocalCacheFileName } from '../common';
 import type { RepoCacheRecord } from '../schema';
 import { RepoCacheBase } from './base';
 
@@ -54,14 +57,22 @@ export class RepoCacheS3 extends RepoCacheBase {
 
   async write(data: RepoCacheRecord): Promise<void> {
     const cacheFileName = this.getCacheFileName();
+    const stringifiedCache = JSON.stringify(data);
     const s3Params: PutObjectCommandInput = {
       Bucket: this.bucket,
       Key: cacheFileName,
-      Body: JSON.stringify(data),
+      Body: stringifiedCache,
       ContentType: 'application/json',
     };
     try {
       await this.s3Client.send(new PutObjectCommand(s3Params));
+      if (is.nonEmptyString(process.env.RENOVATE_X_REPO_CACHE_FORCE_LOCAL)) {
+        const cacheLocalFileName = getLocalCacheFileName(
+          this.platform,
+          this.repository,
+        );
+        await outputCacheFile(cacheLocalFileName, stringifiedCache);
+      }
     } catch (err) {
       logger.warn({ err }, 'RepoCacheS3.write() - failure');
     }