feat(http): Etag support

lib/util/http/bitbucket-server.ts

@@ -1,5 +1,10 @@
 import { resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+  HttpOptions,
+  HttpRequestOptions,
+  HttpResponse,
+  InternalHttpOptions,
+} from './types';
 import { Http } from '.';
 
 let baseUrl: string;
@@ -14,7 +19,7 @@ export class BitbucketServerHttp extends Http {
 
   protected override request<T>(
     path: string,
-    options?: InternalHttpOptions
+    options?: InternalHttpOptions & HttpRequestOptions<T>
   ): Promise<HttpResponse<T>> {
     const url = resolveBaseUrl(baseUrl, path);
     const opts = {

lib/util/http/bitbucket.ts

@@ -1,7 +1,7 @@
 import is from '@sindresorhus/is';
 import type { PagedResult } from '../../modules/platform/bitbucket/types';
 import { parseUrl, resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse } from './types';
+import type { HttpOptions, HttpRequestOptions, HttpResponse } from './types';
 import { Http } from '.';
 
 let baseUrl = 'https://api.bitbucket.org/';
@@ -21,13 +21,15 @@ export class BitbucketHttp extends Http<BitbucketHttpOptions> {
 
   protected override async request<T>(
     path: string,
-    options?: BitbucketHttpOptions
+    options?: BitbucketHttpOptions & HttpRequestOptions<T>
   ): Promise<HttpResponse<T>> {
     const opts = { baseUrl, ...options };
 
     const result = await super.request<T>(path, opts);
 
     if (opts.paginate && isPagedResult(result.body)) {
+      delete opts.etagCache;
+
       const resultBody = result.body as PagedResult<T>;
 
       let nextPage = getPageFromURL(resultBody.next);
@@ -42,7 +44,7 @@ export class BitbucketHttp extends Http<BitbucketHttpOptions> {
 
         const nextResult = await super.request<PagedResult<T>>(
           nextPath,
-          options
+          options as BitbucketHttpOptions
         );
 
         resultBody.values.push(...nextResult.body.values);

lib/util/http/gitea.ts

@@ -1,6 +1,11 @@
 import is from '@sindresorhus/is';
 import { resolveBaseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+  HttpOptions,
+  HttpRequestOptions,
+  HttpResponse,
+  InternalHttpOptions,
+} from './types';
 import { Http } from '.';
 
 let baseUrl: string;
@@ -36,7 +41,7 @@ export class GiteaHttp extends Http<GiteaHttpOptions> {
 
   protected override async request<T>(
     path: string,
-    options?: InternalHttpOptions & GiteaHttpOptions
+    options?: InternalHttpOptions & GiteaHttpOptions & HttpRequestOptions<T>
   ): Promise<HttpResponse<T>> {
     const resolvedUrl = resolveUrl(path, options?.baseUrl ?? baseUrl);
     const opts = {

lib/util/http/github.ts

@@ -19,6 +19,7 @@ import type { GotLegacyError } from './legacy';
 import type {
   GraphqlOptions,
   HttpOptions,
+  HttpRequestOptions,
   HttpResponse,
   InternalHttpOptions,
 } from './types';
@@ -271,7 +272,7 @@ export class GithubHttp extends Http<GithubHttpOptions> {
 
   protected override async request<T>(
     url: string | URL,
-    options?: InternalHttpOptions & GithubHttpOptions,
+    options?: InternalHttpOptions & GithubHttpOptions & HttpRequestOptions<T>,
     okToRetry = true
   ): Promise<HttpResponse<T>> {
     const opts: GithubHttpOptions = {

lib/util/http/gitlab.ts

@@ -2,7 +2,12 @@ import is from '@sindresorhus/is';
 import { logger } from '../../logger';
 import { ExternalHostError } from '../../types/errors/external-host-error';
 import { parseLinkHeader, parseUrl } from '../url';
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+  HttpOptions,
+  HttpRequestOptions,
+  HttpResponse,
+  InternalHttpOptions,
+} from './types';
 import { Http } from '.';
 
 let baseUrl = 'https://gitlab.com/api/v4/';
@@ -21,7 +26,7 @@ export class GitlabHttp extends Http<GitlabHttpOptions> {
 
   protected override async request<T>(
     url: string | URL,
-    options?: InternalHttpOptions & GitlabHttpOptions
+    options?: InternalHttpOptions & GitlabHttpOptions & HttpRequestOptions<T>
   ): Promise<HttpResponse<T>> {
     const opts = {
       baseUrl,

lib/util/http/index.spec.ts

@@ -425,4 +425,81 @@ describe('util/http/index', () => {
       expect(t2 - t1).toBeGreaterThanOrEqual(4000);
     });
   });
+
+  describe('Etag caching', () => {
+    beforeEach(() => {
+      jest.resetAllMocks();
+      memCache.init();
+    });
+
+    afterEach(() => {
+      memCache.reset();
+    });
+
+    type FooBar = { foo: string; bar: string };
+
+    it('returns cached data for status=304', async () => {
+      const data: FooBar = { foo: 'foo', bar: 'bar' };
+      httpMock
+        .scope(baseUrl, { reqheaders: { 'If-None-Match': 'foobar' } })
+        .get('/foo')
+        .reply(304);
+
+      const res = await http.getJson<FooBar>(`/foo`, {
+        baseUrl,
+        etagCache: {
+          etag: 'foobar',
+          data,
+        },
+      });
+
+      expect(res.statusCode).toBe(304);
+      expect(res.body).toEqual(data);
+      expect(res.body).not.toBe(data);
+    });
+
+    it('returns new data for status=200', async () => {
+      const oldData: FooBar = { foo: 'foo', bar: 'bar' };
+      const newData: FooBar = { foo: 'FOO', bar: 'BAR' };
+      httpMock
+        .scope(baseUrl, { reqheaders: { 'If-None-Match': 'foobar' } })
+        .get('/foo')
+        .reply(200, newData);
+
+      const res = await http.getJson<FooBar>(`/foo`, {
+        baseUrl,
+        etagCache: {
+          etag: 'foobar',
+          data: oldData,
+        },
+      });
+
+      expect(res.statusCode).toBe(200);
+      expect(res.body).toEqual(newData);
+      expect(res.body).not.toBe(newData);
+    });
+
+    it('throws if old data does not match the schema', async () => {
+      const FooBar = z.object({ foo: z.string(), bar: z.string() });
+      const data = { foo: 'foo' };
+      httpMock
+        .scope(baseUrl, { reqheaders: { 'If-None-Match': 'foobar' } })
+        .get('/foo')
+        .reply(304);
+
+      await expect(
+        http.getJson(
+          `/foo`,
+          {
+            baseUrl,
+            etagCache: {
+              etag: 'foobar',
+              data: data as never,
+            },
+          },
+          FooBar
+        )
+      ).rejects.toThrow(z.ZodError);
+    });
+  });
 });

lib/util/http/index.ts

@@ -18,6 +18,7 @@
   GotJSONOptions,
   GotOptions,
   HttpOptions,
+  HttpRequestOptions,
   HttpResponse,
   InternalHttpOptions,
   RequestStats,
@@ -28,7 +29,7 @@
 export { RequestError as HttpError };
 
 type JsonArgs<
-  Opts extends HttpOptions,
+  Opts extends HttpOptions & HttpRequestOptions<ResT>,
   ResT = unknown,
   Schema extends ZodType<ResT> = ZodType<ResT>
 > = {
@@ -120,7 +121,7 @@
 
   protected async request<T>(
     requestUrl: string | URL,
-    httpOptions: InternalHttpOptions = {}
+    httpOptions: InternalHttpOptions & HttpRequestOptions<T> = {}
   ): Promise<HttpResponse<T>> {
     let url = requestUrl.toString();
     if (httpOptions?.baseUrl) {
@@ -136,6 +137,17 @@
       httpOptions
     );
 
+    const etagCache =
+      httpOptions.etagCache && options.method === 'get'
+        ? httpOptions.etagCache
+        : null;
+    if (etagCache) {
+      options.headers = {
+        ...options.headers,
+        'If-None-Match': etagCache.etag,
+      };
+    }
+
     if (process.env.NODE_ENV === 'test') {
       options.retry = 0;
     }
@@ -203,9 +215,14 @@
     }
 
     try {
-      const res = await resPromise;
+      const res = memCacheKey
+        ? cloneResponse(await resPromise)
+        : await resPromise;
       res.authorization = !!options?.headers?.authorization;
-      return cloneResponse(res);
+      if (etagCache && res.statusCode === 304) {
+        res.body = clone(etagCache.data);
+      }
+      return res;
     } catch (err) {
       const { abortOnError, abortIgnoreStatusCodes } = options;
       if (abortOnError && !abortIgnoreStatusCodes?.includes(err.statusCode)) {
@@ -215,7 +232,10 @@
     }
   }
 
-  get(url: string, options: HttpOptions = {}): Promise<HttpResponse> {
+  get(
+    url: string,
+    options: HttpOptions & HttpRequestOptions<string> = {}
+  ): Promise<HttpResponse> {
     return this.request<string>(url, options);
   }
 
@@ -281,19 +301,22 @@
     return res;
   }
 
-  getJson<ResT>(url: string, options?: Opts): Promise<HttpResponse<ResT>>;
+  getJson<ResT>(
+    url: string,
+    options?: Opts & HttpRequestOptions<ResT>
+  ): Promise<HttpResponse<ResT>>;
   getJson<ResT, Schema extends ZodType<ResT> = ZodType<ResT>>(
     url: string,
     schema: Schema
   ): Promise<HttpResponse<Infer<Schema>>>;
   getJson<ResT, Schema extends ZodType<ResT> = ZodType<ResT>>(
     url: string,
-    options: Opts,
+    options: Opts & HttpRequestOptions<Infer<Schema>>,
     schema: Schema
   ): Promise<HttpResponse<Infer<Schema>>>;
   getJson<ResT = unknown, Schema extends ZodType<ResT> = ZodType<ResT>>(
     arg1: string,
-    arg2?: Opts | Schema,
+    arg2?: (Opts & HttpRequestOptions<ResT>) | Schema,
     arg3?: Schema
   ): Promise<HttpResponse<ResT>> {
     const args = this.resolveArgs<ResT>(arg1, arg2, arg3);

lib/util/http/jira.ts

@@ -1,4 +1,9 @@
-import type { HttpOptions, HttpResponse, InternalHttpOptions } from './types';
+import type {
+  HttpOptions,
+  HttpRequestOptions,
+  HttpResponse,
+  InternalHttpOptions,
+} from './types';
 import { Http } from '.';
 
 let baseUrl: string;
@@ -14,7 +19,7 @@
 
   protected override request<T>(
     url: string | URL,
-    options?: InternalHttpOptions
+    options?: InternalHttpOptions & HttpRequestOptions<T>
   ): Promise<HttpResponse<T>> {
     const opts = { baseUrl, ...options };
     return super.request<T>(url, opts);

lib/util/http/types.ts

@@ -65,6 +65,15 @@ export interface HttpOptions {
   memCache?: boolean;
 }
 
+export interface EtagCache<T = any> {
+  etag: string;
+  data: T;
+}
+
+export interface HttpRequestOptions<T = any> {
+  etagCache?: EtagCache<T>;
+}
+
 export interface InternalHttpOptions extends HttpOptions {
   json?: HttpOptions['body'];
   responseType?: 'json' | 'buffer';