diff --git a/lib/modules/datasource/npm/get.ts b/lib/modules/datasource/npm/get.ts index 33f5da96829bda..0848f5fdfc01d6 100644 --- a/lib/modules/datasource/npm/get.ts +++ b/lib/modules/datasource/npm/get.ts @@ -10,6 +10,7 @@ import * as packageCache from '../../../util/cache/package'; import type { Http } from '../../../util/http'; import type { HttpOptions } from '../../../util/http/types'; import { regEx } from '../../../util/regex'; +import { HttpCacheStats } from '../../../util/stats'; import { joinUrlParts } from '../../../util/url'; import type { Release, ReleaseResult } from '../types'; import type { CachedReleaseResult, NpmResponse } from './types'; @@ -91,10 +92,13 @@ export async function getDependency( ); if (softExpireAt.isValid && softExpireAt > DateTime.local()) { logger.trace('Cached result is not expired - reusing'); + HttpCacheStats.incLocalHits(packageUrl); delete cachedResult.cacheData; return cachedResult; } + logger.trace('Cached result is soft expired'); + HttpCacheStats.incLocalMisses(packageUrl); } else { logger.trace( `Package cache for npm package "${packageName}" is from an old revision - discarding`, @@ -127,6 +131,7 @@ export async function getDependency( const raw = await http.getJson(packageUrl, options); if (cachedResult?.cacheData && raw.statusCode === 304) { logger.trace(`Cached npm result for ${packageName} is revalidated`); + HttpCacheStats.incRemoteHits(packageUrl); cachedResult.cacheData.softExpireAt = softExpireAt; await packageCache.set( cacheNamespace, @@ -137,6 +142,7 @@ export async function getDependency( delete cachedResult.cacheData; return cachedResult; } + HttpCacheStats.incRemoteMisses(packageUrl); const etag = raw.headers.etag; const res = raw.body; if (!res.versions || !Object.keys(res.versions).length) { diff --git a/lib/util/http/index.ts b/lib/util/http/index.ts index 107f4726344928..d6139238eff4d7 100644 --- a/lib/util/http/index.ts +++ b/lib/util/http/index.ts @@ -11,7 +11,11 @@ import { getCache } from '../cache/repository'; import { clone } from '../clone'; import { hash } from '../hash'; import { type AsyncResult, Result } from '../result'; -import { type HttpRequestStatsDataPoint, HttpStats } from '../stats'; +import { + HttpCacheStats, + type HttpRequestStatsDataPoint, + HttpStats, +} from '../stats'; import { resolveBaseUrl } from '../url'; import { applyAuthorization, removeAuthorization } from './auth'; import { hooks } from './hooks'; @@ -279,6 +283,7 @@ export class Http { logger.debug( `http cache: saving ${url} (etag=${resCopy.headers.etag}, lastModified=${resCopy.headers['last-modified']})`, ); + HttpCacheStats.incRemoteMisses(url); cache.httpCache[url] = { etag: resCopy.headers.etag, httpResponse: copyResponse(res, deepCopyNeeded), @@ -290,6 +295,7 @@ export class Http { logger.debug( `http cache: Using cached response: ${url} from ${cache.httpCache[url].timeStamp}`, ); + HttpCacheStats.incRemoteHits(url); const cacheCopy = copyResponse( cache.httpCache[url].httpResponse, deepCopyNeeded, diff --git a/lib/util/stats.spec.ts b/lib/util/stats.spec.ts index f23bfe35d6e3a1..4ea3c275855af6 100644 --- a/lib/util/stats.spec.ts +++ b/lib/util/stats.spec.ts @@ -1,6 +1,7 @@ import { logger } from '../../test/util'; import * as memCache from './cache/memory'; import { + HttpCacheStats, HttpStats, LookupStats, PackageCacheStats, @@ -455,4 +456,79 @@ describe('util/stats', () => { }); }); }); + + describe('HttpCacheStats', () => { + it('returns empty data', () => { + const res = HttpCacheStats.getData(); + expect(res).toEqual({}); + }); + + it('ignores wrong url', () => { + HttpCacheStats.incLocalHits(''); + expect(HttpCacheStats.getData()).toEqual({}); + }); + + it('writes data points', () => { + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/bar'); + HttpCacheStats.incRemoteHits('https://example.com/bar'); + HttpCacheStats.incRemoteMisses('https://example.com/bar'); + + const res = HttpCacheStats.getData(); + + expect(res).toEqual({ + 'https://example.com/bar': { + localHits: 0, + localMisses: 1, + localTotal: 1, + remoteHits: 1, + remoteMisses: 1, + remoteTotal: 2, + }, + 'https://example.com/foo': { + localHits: 2, + localMisses: 1, + localTotal: 3, + remoteHits: 0, + remoteMisses: 0, + remoteTotal: 0, + }, + }); + }); + + it('prints report', () => { + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalHits('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/foo'); + HttpCacheStats.incLocalMisses('https://example.com/bar'); + HttpCacheStats.incRemoteHits('https://example.com/bar'); + HttpCacheStats.incRemoteMisses('https://example.com/bar'); + + HttpCacheStats.report(); + + expect(logger.logger.debug).toHaveBeenCalledTimes(1); + const [data, msg] = logger.logger.debug.mock.calls[0]; + expect(msg).toBe('HTTP cache statistics'); + expect(data).toEqual({ + 'https://example.com/bar': { + localHits: 0, + localMisses: 1, + localTotal: 1, + remoteHits: 1, + remoteMisses: 1, + remoteTotal: 2, + }, + 'https://example.com/foo': { + localHits: 2, + localMisses: 1, + localTotal: 3, + remoteHits: 0, + remoteMisses: 0, + remoteTotal: 0, + }, + }); + }); + }); }); diff --git a/lib/util/stats.ts b/lib/util/stats.ts index 6508f49d834551..c0e6996e360db5 100644 --- a/lib/util/stats.ts +++ b/lib/util/stats.ts @@ -235,3 +235,99 @@ export class HttpStats { logger.debug({ urls, hosts, requests }, 'HTTP statistics'); } } + +interface HttpCacheHostStatsData { + localHits: number; + localMisses: number; + localTotal: number; + remoteHits: number; + remoteMisses: number; + remoteTotal: number; +} + +type HttpCacheStatsData = Record; + +export class HttpCacheStats { + static getData(): HttpCacheStatsData { + return memCache.get('http-cache-stats') ?? {}; + } + + static read(key: string): HttpCacheHostStatsData { + return ( + this.getData()?.[key] ?? { + localHits: 0, + localMisses: 0, + localTotal: 0, + remoteHits: 0, + remoteMisses: 0, + remoteTotal: 0, + } + ); + } + + static write(key: string, data: HttpCacheHostStatsData): void { + const stats = memCache.get('http-cache-stats') ?? {}; + stats[key] = data; + memCache.set('http-cache-stats', stats); + } + + static getBaseUrl(url: string): string | null { + const parsedUrl = parseUrl(url); + if (!parsedUrl) { + logger.debug({ url }, 'Failed to parse URL during cache stats'); + return null; + } + const { origin, pathname } = parsedUrl; + const baseUrl = `${origin}${pathname}`; + return baseUrl; + } + + static incLocalHits(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.localHits += 1; + stats.localTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static incLocalMisses(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.localMisses += 1; + stats.localTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static incRemoteHits(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.remoteHits += 1; + stats.remoteTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static incRemoteMisses(url: string): void { + const baseUrl = HttpCacheStats.getBaseUrl(url); + if (baseUrl) { + const host = baseUrl; + const stats = HttpCacheStats.read(host); + stats.remoteMisses += 1; + stats.remoteTotal += 1; + HttpCacheStats.write(host, stats); + } + } + + static report(): void { + const stats = HttpCacheStats.getData(); + logger.debug(stats, 'HTTP cache statistics'); + } +} diff --git a/lib/workers/repository/index.ts b/lib/workers/repository/index.ts index 01c0658eb3e79b..fa7fa880e833eb 100644 --- a/lib/workers/repository/index.ts +++ b/lib/workers/repository/index.ts @@ -19,7 +19,12 @@ import { clearDnsCache, printDnsStats } from '../../util/http/dns'; import * as queue from '../../util/http/queue'; import * as throttle from '../../util/http/throttle'; import { addSplit, getSplits, splitInit } from '../../util/split'; -import { HttpStats, LookupStats, PackageCacheStats } from '../../util/stats'; +import { + HttpCacheStats, + HttpStats, + LookupStats, + PackageCacheStats, +} from '../../util/stats'; import { setBranchCache } from './cache'; import { extractRepoProblems } from './common'; import { ensureDependencyDashboard } from './dependency-dashboard'; @@ -126,6 +131,7 @@ export async function renovateRepository( logger.debug(splits, 'Repository timing splits (milliseconds)'); PackageCacheStats.report(); HttpStats.report(); + HttpCacheStats.report(); LookupStats.report(); printDnsStats(); clearDnsCache();