refactor: Extract "HttpStats" utility class (#27944)

lib/util/http/index.ts

@@ -11,6 +11,7 @@ import { getCache } from '../cache/repository';
 import { clone } from '../clone';
 import { hash } from '../hash';
 import { type AsyncResult, Result } from '../result';
+import { type HttpRequestStatsDataPoint, HttpStats } from '../stats';
 import { resolveBaseUrl } from '../url';
 import { applyAuthorization, removeAuthorization } from './auth';
 import { hooks } from './hooks';
@@ -26,7 +27,6 @@ import type {
   HttpRequestOptions,
   HttpResponse,
   InternalHttpOptions,
-  RequestStats,
 } from './types';
 // TODO: refactor code to remove this (#9651)
 import './legacy';
@@ -76,6 +76,8 @@ function applyDefaultHeaders(options: Options): void {
   };
 }
 
+type QueueStatsData = Pick<HttpRequestStatsDataPoint, 'queueMs'>;
+
 // Note on types:
 // options.requestType can be either 'json' or 'buffer', but `T` should be
 // `Buffer` in the latter case.
@@ -84,7 +86,7 @@ function applyDefaultHeaders(options: Options): void {
 async function gotTask<T>(
   url: string,
   options: SetRequired<GotOptions, 'method'>,
-  requestStats: Omit<RequestStats, 'duration' | 'statusCode'>,
+  queueStats: QueueStatsData,
 ): Promise<HttpResponse<T>> {
   logger.trace({ url, options }, 'got request');
 
@@ -119,9 +121,13 @@ async function gotTask<T>(
 
     throw error;
   } finally {
-    const httpRequests = memCache.get<RequestStats[]>('http-requests') || [];
-    httpRequests.push({ ...requestStats, duration, statusCode });
-    memCache.set('http-requests', httpRequests);
+    HttpStats.write({
+      method: options.method,
+      url,
+      reqMs: duration,
+      queueMs: queueStats.queueMs,
+      status: statusCode,
+    });
   }
 }
 
@@ -236,12 +242,8 @@ export class Http<Opts extends HttpOptions = HttpOptions> {
       }
       const startTime = Date.now();
       const httpTask: GotTask<T> = () => {
-        const queueDuration = Date.now() - startTime;
-        return gotTask(url, options, {
-          method: options.method,
-          url,
-          queueDuration,
-        });
+        const queueMs = Date.now() - startTime;
+        return gotTask(url, options, { queueMs });
       };
 
       const throttle = this.getThrottle(url);

lib/util/stats.spec.ts

@@ -1,6 +1,11 @@
 import { logger } from '../../test/util';
 import * as memCache from './cache/memory';
-import { LookupStats, PackageCacheStats, makeTimingReport } from './stats';
+import {
+  HttpStats,
+  LookupStats,
+  PackageCacheStats,
+  makeTimingReport,
+} from './stats';
 
 describe('util/stats', () => {
   beforeEach(() => {
@@ -223,4 +228,231 @@ describe('util/stats', () => {
       });
     });
   });
+
+  describe('HttpStats', () => {
+    it('returns empty report', () => {
+      const res = HttpStats.getReport();
+      expect(res).toEqual({
+        hostRequests: {},
+        hosts: {},
+        rawRequests: [],
+        requests: 0,
+        urls: {},
+      });
+    });
+
+    it('writes data points', () => {
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 100,
+        queueMs: 10,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 200,
+        queueMs: 20,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/bar',
+        reqMs: 400,
+        queueMs: 40,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 800,
+        queueMs: 80,
+        status: 404,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: '<invalid>',
+        reqMs: 100,
+        queueMs: 100,
+        status: 400,
+      });
+
+      const res = HttpStats.getReport();
+
+      expect(res).toEqual({
+        hostRequests: {
+          'example.com': [
+            {
+              method: 'GET',
+              queueMs: 40,
+              reqMs: 400,
+              status: 200,
+              url: 'https://example.com/bar',
+            },
+            {
+              method: 'GET',
+              queueMs: 10,
+              reqMs: 100,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 20,
+              reqMs: 200,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 80,
+              reqMs: 800,
+              status: 404,
+              url: 'https://example.com/foo',
+            },
+          ],
+        },
+        hosts: {
+          'example.com': {
+            count: 4,
+            queueAvgMs: 38,
+            queueMaxMs: 80,
+            queueMedianMs: 40,
+            reqAvgMs: 375,
+            reqMaxMs: 800,
+            reqMedianMs: 400,
+          },
+        },
+        rawRequests: [
+          'GET https://example.com/bar 200 400 40',
+          'GET https://example.com/foo 200 100 10',
+          'GET https://example.com/foo 200 200 20',
+          'GET https://example.com/foo 404 800 80',
+        ],
+        requests: 5,
+        urls: {
+          'https://example.com/bar': {
+            GET: {
+              '200': 1,
+            },
+          },
+          'https://example.com/foo': {
+            GET: {
+              '200': 2,
+              '404': 1,
+            },
+          },
+        },
+      });
+    });
+
+    it('logs report', () => {
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 100,
+        queueMs: 10,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 200,
+        queueMs: 20,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/bar',
+        reqMs: 400,
+        queueMs: 40,
+        status: 200,
+      });
+      HttpStats.write({
+        method: 'GET',
+        url: 'https://example.com/foo',
+        reqMs: 800,
+        queueMs: 80,
+        status: 404,
+      });
+
+      HttpStats.report();
+
+      expect(logger.logger.trace).toHaveBeenCalledTimes(1);
+      const [traceData, traceMsg] = logger.logger.trace.mock.calls[0];
+      expect(traceMsg).toBe('HTTP full statistics');
+      expect(traceData).toEqual({
+        rawRequests: [
+          'GET https://example.com/bar 200 400 40',
+          'GET https://example.com/foo 200 100 10',
+          'GET https://example.com/foo 200 200 20',
+          'GET https://example.com/foo 404 800 80',
+        ],
+        hostRequests: {
+          'example.com': [
+            {
+              method: 'GET',
+              queueMs: 40,
+              reqMs: 400,
+              status: 200,
+              url: 'https://example.com/bar',
+            },
+            {
+              method: 'GET',
+              queueMs: 10,
+              reqMs: 100,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 20,
+              reqMs: 200,
+              status: 200,
+              url: 'https://example.com/foo',
+            },
+            {
+              method: 'GET',
+              queueMs: 80,
+              reqMs: 800,
+              status: 404,
+              url: 'https://example.com/foo',
+            },
+          ],
+        },
+      });
+
+      expect(logger.logger.debug).toHaveBeenCalledTimes(1);
+      const [debugData, debugMsg] = logger.logger.debug.mock.calls[0];
+      expect(debugMsg).toBe('HTTP statistics');
+      expect(debugData).toEqual({
+        hosts: {
+          'example.com': {
+            count: 4,
+            queueAvgMs: 38,
+            queueMaxMs: 80,
+            queueMedianMs: 40,
+            reqAvgMs: 375,
+            reqMaxMs: 800,
+            reqMedianMs: 400,
+          },
+        },
+        requests: 4,
+        urls: {
+          'https://example.com/bar': {
+            GET: {
+              '200': 1,
+            },
+          },
+          'https://example.com/foo': {
+            GET: {
+              '200': 2,
+              '404': 1,
+            },
+          },
+        },
+      });
+    });
+  });
 });