feat(http): Add pluggable HTTP cache implementation (#27998)

lib/util/cache/repository/types.ts

@@ -6,7 +6,6 @@ import type {
 import type { PackageFile } from '../../../modules/manager/types';
 import type { RepoInitConfig } from '../../../workers/repository/init/types';
 import type { PrBlockedBy } from '../../../workers/types';
-import type { HttpResponse } from '../../http/types';
 
 export interface BaseBranchCache {
   sha: string; // branch commit sha
@@ -123,16 +122,9 @@ export interface BranchCache {
   result?: string;
 }
 
-export interface HttpCache {
-  etag?: string;
-  httpResponse: HttpResponse<unknown>;
-  lastModified?: string;
-  timeStamp: string;
-}
-
 export interface RepoCacheData {
   configFileName?: string;
-  httpCache?: Record<string, HttpCache>;
+  httpCache?: Record<string, unknown>;
   semanticCommits?: 'enabled' | 'disabled';
   branches?: BranchCache[];
   init?: RepoInitConfig;

lib/util/http/cache/abstract-http-cache-provider.ts

@@ -0,0 +1,94 @@
+import { logger } from '../../../logger';
+import { HttpCacheStats } from '../../stats';
+import type { GotOptions, HttpResponse } from '../types';
+import { copyResponse } from '../util';
+import { HttpCacheSchema } from './schema';
+import type { HttpCache, HttpCacheProvider } from './types';
+
+export abstract class AbstractHttpCacheProvider implements HttpCacheProvider {
+  protected abstract load(url: string): Promise<unknown>;
+  protected abstract persist(url: string, data: HttpCache): Promise<void>;
+
+  async get(url: string): Promise<HttpCache | null> {
+    const cache = await this.load(url);
+    const httpCache = HttpCacheSchema.parse(cache);
+    if (!httpCache) {
+      return null;
+    }
+
+    return httpCache as HttpCache;
+  }
+
+  async setCacheHeaders<T extends Pick<GotOptions, 'headers'>>(
+    url: string,
+    opts: T,
+  ): Promise<void> {
+    const httpCache = await this.get(url);
+    if (!httpCache) {
+      return;
+    }
+
+    opts.headers ??= {};
+
+    if (httpCache.etag) {
+      opts.headers['If-None-Match'] = httpCache.etag;
+    }
+
+    if (httpCache.lastModified) {
+      opts.headers['If-Modified-Since'] = httpCache.lastModified;
+    }
+  }
+
+  async wrapResponse<T>(
+    url: string,
+    resp: HttpResponse<T>,
+  ): Promise<HttpResponse<T>> {
+    if (resp.statusCode === 200) {
+      const etag = resp.headers?.['etag'];
+      const lastModified = resp.headers?.['last-modified'];
+
+      HttpCacheStats.incRemoteMisses(url);
+
+      const httpResponse = copyResponse(resp, true);
+      const timestamp = new Date().toISOString();
+
+      const newHttpCache = HttpCacheSchema.parse({
+        etag,
+        lastModified,
+        httpResponse,
+        timestamp,
+      });
+      if (newHttpCache) {
+        logger.debug(
+          `http cache: saving ${url} (etag=${etag}, lastModified=${lastModified})`,
+        );
+        await this.persist(url, newHttpCache as HttpCache);
+      } else {
+        logger.debug(`http cache: failed to persist cache for ${url}`);
+      }
+
+      return resp;
+    }
+
+    if (resp.statusCode === 304) {
+      const httpCache = await this.get(url);
+      if (!httpCache) {
+        return resp;
+      }
+
+      const timestamp = httpCache.timestamp;
+      logger.debug(
+        `http cache: Using cached response: ${url} from ${timestamp}`,
+      );
+      HttpCacheStats.incRemoteHits(url);
+      const cachedResp = copyResponse(
+        httpCache.httpResponse as HttpResponse<T>,
+        true,
+      );
+      cachedResp.authorization = resp.authorization;
+      return cachedResp;
+    }
+
+    return resp;
+  }
+}

lib/util/http/cache/repository-http-cache-provider.spec.ts

@@ -0,0 +1,152 @@
+import { Http } from '..';
+import * as httpMock from '../../../../test/http-mock';
+import { logger } from '../../../../test/util';
+import { getCache, resetCache } from '../../cache/repository';
+import { repoCacheProvider } from './repository-http-cache-provider';
+
+describe('util/http/cache/repository-http-cache-provider', () => {
+  beforeEach(() => {
+    resetCache();
+  });
+
+  const http = new Http('test', {
+    cacheProvider: repoCacheProvider,
+  });
+
+  it('reuses data with etag', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+    const res1 = await http.getJson('https://example.com/foo/bar');
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar');
+    expect(res2).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+  });
+
+  it('reuses data with last-modified', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope
+      .get('/foo/bar')
+      .reply(
+        200,
+        { msg: 'Hello, world!' },
+        { 'last-modified': 'Mon, 01 Jan 2000 00:00:00 GMT' },
+      );
+    const res1 = await http.getJson('https://example.com/foo/bar');
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar');
+    expect(res2).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+  });
+
+  it('uses older cache format', async () => {
+    const repoCache = getCache();
+    repoCache.httpCache = {
+      'https://example.com/foo/bar': {
+        etag: '123',
+        lastModified: 'Mon, 01 Jan 2000 00:00:00 GMT',
+        httpResponse: { statusCode: 200, body: { msg: 'Hello, world!' } },
+        timeStamp: new Date().toISOString(),
+      },
+    };
+    httpMock.scope('https://example.com').get('/foo/bar').reply(304);
+
+    const res = await http.getJson('https://example.com/foo/bar');
+
+    expect(res).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+  });
+
+  it('reports if cache could not be persisted', async () => {
+    httpMock
+      .scope('https://example.com')
+      .get('/foo/bar')
+      .reply(200, { msg: 'Hello, world!' });
+
+    await http.getJson('https://example.com/foo/bar');
+
+    expect(logger.logger.debug).toHaveBeenCalledWith(
+      'http cache: failed to persist cache for https://example.com/foo/bar',
+    );
+  });
+
+  it('handles abrupt cache reset', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+    const res1 = await http.getJson('https://example.com/foo/bar');
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: false,
+    });
+
+    resetCache();
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar');
+    expect(res2).toMatchObject({
+      statusCode: 304,
+      authorization: false,
+    });
+  });
+
+  it('bypasses for statuses other than 200 and 304', async () => {
+    const scope = httpMock.scope('https://example.com');
+    scope.get('/foo/bar').reply(203);
+
+    const res = await http.getJson('https://example.com/foo/bar');
+
+    expect(res).toMatchObject({
+      statusCode: 203,
+      authorization: false,
+    });
+  });
+
+  it('supports authorization', async () => {
+    const scope = httpMock.scope('https://example.com');
+
+    scope.get('/foo/bar').reply(200, { msg: 'Hello, world!' }, { etag: '123' });
+    const res1 = await http.getJson('https://example.com/foo/bar', {
+      headers: { authorization: 'Bearer 123' },
+    });
+    expect(res1).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: true,
+    });
+
+    scope.get('/foo/bar').reply(304);
+    const res2 = await http.getJson('https://example.com/foo/bar', {
+      headers: { authorization: 'Bearer 123' },
+    });
+    expect(res2).toMatchObject({
+      statusCode: 200,
+      body: { msg: 'Hello, world!' },
+      authorization: true,
+    });
+  });
+});

lib/util/http/cache/repository-http-cache-provider.ts

@@ -0,0 +1,20 @@
+import { getCache } from '../../cache/repository';
+import { AbstractHttpCacheProvider } from './abstract-http-cache-provider';
+import type { HttpCache } from './types';
+
+export class RepositoryHttpCacheProvider extends AbstractHttpCacheProvider {
+  override load(url: string): Promise<unknown> {
+    const cache = getCache();
+    cache.httpCache ??= {};
+    return Promise.resolve(cache.httpCache[url]);
+  }
+
+  override persist(url: string, data: HttpCache): Promise<void> {
+    const cache = getCache();
+    cache.httpCache ??= {};
+    cache.httpCache[url] = data;
+    return Promise.resolve();
+  }
+}
+
+export const repoCacheProvider = new RepositoryHttpCacheProvider();

lib/util/http/cache/schema.ts

@@ -0,0 +1,34 @@
+import { z } from 'zod';
+
+const invalidFieldsMsg =
+  'Cache object should have `etag` or `lastModified` fields';
+
+export const HttpCacheSchema = z
+  .object({
+    // TODO: remove this migration part during the Christmas eve 2024
+    timeStamp: z.string().optional(),
+    timestamp: z.string().optional(),
+  })
+  .passthrough()
+  .transform((data) => {
+    if (data.timeStamp) {
+      data.timestamp = data.timeStamp;
+      delete data.timeStamp;
+    }
+    return data;
+  })
+  .pipe(
+    z
+      .object({
+        etag: z.string().optional(),
+        lastModified: z.string().optional(),
+        httpResponse: z.unknown(),
+        timestamp: z.string(),
+      })
+      .refine(
+        ({ etag, lastModified }) => etag ?? lastModified,
+        invalidFieldsMsg,
+      ),
+  )
+  .nullable()
+  .catch(null);

lib/util/http/cache/types.ts

@@ -0,0 +1,17 @@
+import type { GotOptions, HttpResponse } from '../types';
+
+export interface HttpCache {
+  etag?: string;
+  lastModified?: string;
+  httpResponse: unknown;
+  timestamp: string;
+}
+
+export interface HttpCacheProvider {
+  setCacheHeaders<T extends Pick<GotOptions, 'headers'>>(
+    url: string,
+    opts: T,
+  ): Promise<void>;
+
+  wrapResponse<T>(url: string, resp: HttpResponse<T>): Promise<HttpResponse<T>>;
+}