feat(datasource/npm): cache etag for reuse (#19823)

docs/usage/self-hosted-configuration.md

@@ -200,6 +200,17 @@ For example:
 }
 ```
 
+## cacheHardTtlMinutes
+
+This experimental feature is used to implement the concept of a "soft" cache expiry for datasources, starting with `npm`.
+It should be set to a non-zero value, recommended to be at least 60 (i.e. one hour).
+
+When this value is set, the `npm` datasource will use the `cacheHardTtlMinutes` value for cache expiry, instead of its default expiry of 15 minutes, which becomes the "soft" expiry value.
+Results which are soft expired are reused in the following manner:
+
+- The `etag` from the cached results will be reused, and may result in a 304 response, meaning cached results are revalidated
+- If an error occurs when querying the `npmjs` registry, then soft expired results will be reused if they are present
+
 ## containerbaseDir
 
 This directory is used to cache downloads when `binarySource=docker` or `binarySource=install`.

lib/config/global.ts

@@ -10,6 +10,7 @@ export class GlobalConfig {
     'allowScripts',
     'binarySource',
     'cacheDir',
+    'cacheHardTtlMinutes',
     'containerbaseDir',
     'customEnvVariables',
     'dockerChildPrefix',

lib/config/options/index.ts

@@ -2217,6 +2217,16 @@ const options: RenovateOptions[] = [
     env: false,
     experimental: true,
   },
+  {
+    name: 'cacheHardTtlMinutes',
+    description:
+      'Maximum duration in minutes to keep datasource cache entries.',
+    type: 'integer',
+    stage: 'repository',
+    default: 0,
+    globalOnly: true,
+    experimental: true,
+  },
   {
     name: 'prBodyDefinitions',
     description: 'Table column definitions to use in PR tables.',

lib/config/types.ts

@@ -114,6 +114,7 @@ export interface RepoGlobalConfig {
   allowScripts?: boolean;
   allowedPostUpgradeCommands?: string[];
   binarySource?: 'docker' | 'global' | 'install' | 'hermit';
+  cacheHardTtlMinutes?: number;
   customEnvVariables?: Record<string, string>;
   dockerChildPrefix?: string;
   dockerImagePrefix?: string;

lib/modules/datasource/npm/get.spec.ts

@@ -1,10 +1,16 @@
 import * as httpMock from '../../../../test/http-mock';
+import { mocked } from '../../../../test/util';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
+import * as _packageCache from '../../../util/cache/package';
 import * as hostRules from '../../../util/host-rules';
 import { Http } from '../../../util/http';
 import { getDependency } from './get';
 import { resolveRegistryUrl, setNpmrc } from './npmrc';
 
+jest.mock('../../../util/cache/package');
+
+const packageCache = mocked(_packageCache);
+
 function getPath(s = ''): string {
   const [x] = s.split('\n');
   const prePath = x.replace(/^.*https:\/\/test\.org/, '');
@@ -463,4 +469,53 @@ describe('modules/datasource/npm/get', () => {
       ]
     `);
   });
+
+  it('returns cached legacy', async () => {
+    packageCache.get.mockResolvedValueOnce({ some: 'result' });
+    const dep = await getDependency(http, 'https://some.url', 'some-package');
+    expect(dep).toMatchObject({ some: 'result' });
+  });
+
+  it('returns unexpired cache', async () => {
+    packageCache.get.mockResolvedValueOnce({
+      some: 'result',
+      cacheData: { softExpireAt: '2099' },
+    });
+    const dep = await getDependency(http, 'https://some.url', 'some-package');
+    expect(dep).toMatchObject({ some: 'result' });
+  });
+
+  it('returns soft expired cache if revalidated', async () => {
+    packageCache.get.mockResolvedValueOnce({
+      some: 'result',
+      cacheData: {
+        softExpireAt: '2020',
+        etag: 'some-etag',
+      },
+    });
+    setNpmrc('registry=https://test.org\n_authToken=XXX');
+
+    httpMock.scope('https://test.org').get('/@neutrinojs%2Freact').reply(304);
+    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
+    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
+    expect(dep).toMatchObject({ some: 'result' });
+  });
+
+  it('returns soft expired cache on npmjs error', async () => {
+    packageCache.get.mockResolvedValueOnce({
+      some: 'result',
+      cacheData: {
+        softExpireAt: '2020',
+        etag: 'some-etag',
+      },
+    });
+
+    httpMock
+      .scope('https://registry.npmjs.org')
+      .get('/@neutrinojs%2Freact')
+      .reply(500);
+    const registryUrl = resolveRegistryUrl('@neutrinojs/react');
+    const dep = await getDependency(http, registryUrl, '@neutrinojs/react');
+    expect(dep).toMatchObject({ some: 'result' });
+  });
 });

lib/modules/datasource/npm/get.ts

@@ -1,13 +1,21 @@
 import url from 'url';
 import is from '@sindresorhus/is';
+import { DateTime } from 'luxon';
+import { GlobalConfig } from '../../../config/global';
 import { logger } from '../../../logger';
 import { ExternalHostError } from '../../../types/errors/external-host-error';
 import * as packageCache from '../../../util/cache/package';
 import type { Http } from '../../../util/http';
+import type { HttpOptions } from '../../../util/http/types';
 import { regEx } from '../../../util/regex';
 import { joinUrlParts } from '../../../util/url';
 import { id } from './common';
-import type { NpmDependency, NpmRelease, NpmResponse } from './types';
+import type {
+  CachedNpmDependency,
+  NpmDependency,
+  NpmRelease,
+  NpmResponse,
+} from './types';
 
 interface PackageSource {
   sourceUrl?: string;
@@ -56,19 +64,57 @@ export async function getDependency(
 
   // Now check the persistent cache
   const cacheNamespace = 'datasource-npm';
-  const cachedResult = await packageCache.get<NpmDependency>(
+  const cachedResult = await packageCache.get<CachedNpmDependency>(
     cacheNamespace,
     packageUrl
   );
-  // istanbul ignore if
   if (cachedResult) {
-    return cachedResult;
+    if (cachedResult.cacheData) {
+      const softExpireAt = DateTime.fromISO(
+        cachedResult.cacheData.softExpireAt
+      );
+      if (softExpireAt.isValid && softExpireAt > DateTime.local()) {
+        logger.trace('Cached result is not expired - reusing');
+        delete cachedResult.cacheData;
+        return cachedResult;
+      }
+      logger.trace('Cached result is soft expired');
+    } else {
+      logger.trace('Reusing legacy cached result');
+      return cachedResult;
+    }
+  }
+  const cacheMinutes = process.env.RENOVATE_CACHE_NPM_MINUTES
+    ? parseInt(process.env.RENOVATE_CACHE_NPM_MINUTES, 10)
+    : 15;
+  const softExpireAt = DateTime.local().plus({ minutes: cacheMinutes }).toISO();
+  let { cacheHardTtlMinutes } = GlobalConfig.get();
+  if (!(is.number(cacheHardTtlMinutes) && cacheHardTtlMinutes > cacheMinutes)) {
+    cacheHardTtlMinutes = cacheMinutes;
   }
 
   const uri = url.parse(packageUrl);
 
   try {
-    const raw = await http.getJson<NpmResponse>(packageUrl);
+    const options: HttpOptions = {};
+    if (cachedResult?.cacheData?.etag) {
+      logger.debug('Using cached etag');
+      options.headers = { 'If-None-Match': cachedResult.cacheData.etag };
+    }
+    const raw = await http.getJson<NpmResponse>(packageUrl, options);
+    if (cachedResult?.cacheData && raw.statusCode === 304) {
+      logger.trace('Cached data is unchanged and can be reused');
+      cachedResult.cacheData.softExpireAt = softExpireAt;
+      await packageCache.set(
+        cacheNamespace,
+        packageUrl,
+        cachedResult,
+        cacheHardTtlMinutes
+      );
+      delete cachedResult.cacheData;
+      return cachedResult;
+    }
+    const etag = raw.headers.etag;
     const res = raw.body;
     if (!res.versions || !Object.keys(res.versions).length) {
       // Registry returned a 200 OK but with no versions
@@ -125,9 +171,6 @@ export async function getDependency(
     });
     logger.trace({ dep }, 'dep');
     // serialize first before saving
-    const cacheMinutes = process.env.RENOVATE_CACHE_NPM_MINUTES
-      ? parseInt(process.env.RENOVATE_CACHE_NPM_MINUTES, 10)
-      : 15;
     // TODO: use dynamic detection of public repos instead of a static list (#9587)
     const whitelistedPublicScopes = [
       '@graphql-codegen',
@@ -140,7 +183,13 @@ export async function getDependency(
       (whitelistedPublicScopes.includes(packageName.split('/')[0]) ||
         !packageName.startsWith('@'))
     ) {
-      await packageCache.set(cacheNamespace, packageUrl, dep, cacheMinutes);
+      const cacheData = { softExpireAt, etag };
+      await packageCache.set(
+        cacheNamespace,
+        packageUrl,
+        { ...dep, cacheData },
+        etag ? cacheHardTtlMinutes : cacheMinutes
+      );
     }
     return dep;
   } catch (err) {
@@ -153,6 +202,14 @@ export async function getDependency(
       return null;
     }
     if (uri.host === 'registry.npmjs.org') {
+      if (cachedResult) {
+        logger.warn(
+          { err },
+          'npmjs error, reusing expired cached result instead'
+        );
+        delete cachedResult.cacheData;
+        return cachedResult;
+      }
       // istanbul ignore if
       if (err.name === 'ParseError' && err.body) {
         err.body = 'err.body deleted by Renovate';

lib/modules/datasource/npm/types.ts

@@ -46,4 +46,11 @@ export interface NpmDependency extends ReleaseResult {
   sourceDirectory?: string;
 }
 
+export interface CachedNpmDependency extends NpmDependency {
+  cacheData?: {
+    etag: string | undefined;
+    softExpireAt: string;
+  };
+}
+
 export type Npmrc = Record<string, any>;