feat(repo/cache): add s3 support for user configured folder hierarchy (…

…#18865) Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>
renovatebot · Nov 13, 2022 · 991cc7a · 991cc7a
1 parent d3a239a
commit 991cc7a
Show file tree

Hide file tree

Showing 3 changed files with 97 additions and 6 deletions.
diff --git a/docs/usage/self-hosted-configuration.md b/docs/usage/self-hosted-configuration.md
@@ -669,6 +669,11 @@ Set this to an S3 URI to enable S3 backed repository cache.
     AWS_REGION
 ```
 
+<!-- prettier-ignore -->
+!!! tip
+    If you're storing the repository cache on Amazon S3 then you may set a folder hierarchy as part of `repositoryCacheType`.
+    For example, `repositoryCacheType: 's3://bucket-name/dir1/.../dirN/'`.
+
 ## requireConfig
 
 By default, Renovate needs a Renovate config file in each repository where it runs before it will propose any dependency updates.

diff --git a/lib/util/cache/repository/impl/s3.spec.ts b/lib/util/cache/repository/impl/s3.spec.ts
@@ -18,21 +18,24 @@ import { RepoCacheS3 } from './s3';
 
 function createGetObjectCommandInput(
   repository: string,
-  url: string
+  url: string,
+  folder = ''
 ): GetObjectCommandInput {
+  const platform = GlobalConfig.get('platform')!;
   return {
     Bucket: parseS3Url(url)?.Bucket,
-    Key: `github/${repository}/cache.json`,
+    Key: `${folder}${platform}/${repository}/cache.json`,
   };
 }
 
 function createPutObjectCommandInput(
   repository: string,
   url: string,
-  data: RepoCacheRecord
+  data: RepoCacheRecord,
+  folder = ''
 ): PutObjectCommandInput {
   return {
-    ...createGetObjectCommandInput(repository, url),
+    ...createGetObjectCommandInput(repository, url, folder),
     Body: JSON.stringify(data),
     ContentType: 'application/json',
   };
@@ -76,6 +79,49 @@ describe('util/cache/repository/impl/s3', () => {
     expect(logger.debug).toHaveBeenCalledWith('RepoCacheS3.read() - success');
   });
 
+  it('successfully reads from s3://bucket/dir1/.../dirN/', async () => {
+    const json = '{}';
+    const folder = 'dir1/dir2/dir3/';
+    s3Cache = new RepoCacheS3(
+      repository,
+      '0123456789abcdef',
+      `${url}/${folder}`
+    );
+    s3Mock
+      .on(
+        GetObjectCommand,
+        createGetObjectCommandInput(repository, url, folder)
+      )
+      .resolvesOnce({ Body: Readable.from([json]) });
+    await expect(s3Cache.read()).resolves.toBe(json);
+    expect(logger.warn).toHaveBeenCalledTimes(0);
+    expect(logger.error).toHaveBeenCalledTimes(0);
+    expect(logger.debug).toHaveBeenCalledWith('RepoCacheS3.read() - success');
+  });
+
+  it('appends a missing traling slash to pathname when instantiating RepoCacheS3', async () => {
+    const json = '{}';
+    const pathname = 'dir1/dir2/dir3/file.ext';
+    s3Cache = new RepoCacheS3(
+      repository,
+      '0123456789abcdef',
+      `${url}/${pathname}`
+    );
+    s3Mock
+      .on(
+        GetObjectCommand,
+        createGetObjectCommandInput(repository, url, pathname + '/')
+      )
+      .resolvesOnce({ Body: Readable.from([json]) });
+    await expect(s3Cache.read()).resolves.toBe(json);
+    expect(logger.debug).toHaveBeenCalledWith('RepoCacheS3.read() - success');
+    expect(logger.warn).toHaveBeenCalledTimes(1);
+    expect(logger.warn).toHaveBeenCalledWith(
+      { pathname },
+      'RepoCacheS3.getCacheFolder() - appending missing trailing slash to pathname'
+    );
+  });
+
   it('gets an unexpected response from s3', async () => {
     s3Mock.on(GetObjectCommand, getObjectCommandInput).resolvesOnce({});
     await expect(s3Cache.read()).resolves.toBeNull();
@@ -117,6 +163,27 @@ describe('util/cache/repository/impl/s3', () => {
     expect(logger.warn).toHaveBeenCalledTimes(0);
   });
 
+  it('successfully writes to s3://bucket/dir1/.../dirN/', async () => {
+    const putObjectCommandOutput: PutObjectCommandOutput = {
+      $metadata: { attempts: 1, httpStatusCode: 200, totalRetryDelay: 0 },
+    };
+    const folder = 'dir1/dir2/dir3/';
+    s3Cache = new RepoCacheS3(
+      repository,
+      '0123456789abcdef',
+      `${url}/${folder}`
+    );
+    s3Mock
+      .on(
+        PutObjectCommand,
+        createPutObjectCommandInput(repository, url, repoCache, folder)
+      )
+      .resolvesOnce(putObjectCommandOutput);
+    await expect(s3Cache.write(repoCache)).toResolve();
+    expect(logger.warn).toHaveBeenCalledTimes(0);
+    expect(logger.error).toHaveBeenCalledTimes(0);
+  });
+
   it('fails to write to s3', async () => {
     s3Mock.on(PutObjectCommand, putObjectCommandInput).rejectsOnce(err);
     await expect(s3Cache.write(repoCache)).toResolve();

diff --git a/lib/util/cache/repository/impl/s3.ts b/lib/util/cache/repository/impl/s3.ts
@@ -14,10 +14,13 @@ import { RepoCacheBase } from './base';
 export class RepoCacheS3 extends RepoCacheBase {
   private readonly s3Client;
   private readonly bucket;
+  private readonly dir;
 
   constructor(repository: string, fingerprint: string, url: string) {
     super(repository, fingerprint);
-    this.bucket = parseS3Url(url)?.Bucket;
+    const { Bucket, Key } = parseS3Url(url)!;
+    this.dir = this.getCacheFolder(Key);
+    this.bucket = Bucket;
     this.s3Client = getS3Client();
   }
 
@@ -64,7 +67,23 @@ export class RepoCacheS3 extends RepoCacheBase {
     }
   }
 
+  private getCacheFolder(pathname: string | undefined): string {
+    if (!pathname) {
+      return '';
+    }
+
+    if (pathname.endsWith('/')) {
+      return pathname;
+    }
+
+    logger.warn(
+      { pathname },
+      'RepoCacheS3.getCacheFolder() - appending missing trailing slash to pathname'
+    );
+    return pathname + '/';
+  }
+
   private getCacheFileName(): string {
-    return `${this.platform}/${this.repository}/cache.json`;
+    return `${this.dir}${this.platform}/${this.repository}/cache.json`;
   }
 }