feat(github): long-term datasource caching

lib/modules/datasource/github-releases/cache-base.ts

@@ -0,0 +1,208 @@
+import { DateTime, DurationLike } from 'luxon';
+import { logger } from '../../../logger';
+import * as packageCache from '../../../util/cache/package';
+import type {
+  GithubGraphqlResponse,
+  GithubHttp,
+} from '../../../util/http/github';
+import type { GetReleasesConfig, Release } from '../types';
+import { getApiBaseUrl } from './common';
+
+export interface GithubQueryParams {
+  owner: string;
+  name: string;
+  cursor: string | null;
+  count: number;
+}
+
+interface QueryResponse<T extends FetchedItemBase> {
+  repository: {
+    payload: {
+      nodes: T[];
+      pageInfo: {
+        hasNextPage: boolean;
+        endCursor: string;
+      };
+    };
+  };
+}
+
+export interface FetchedItemBase {
+  version: string;
+}
+
+export interface StoredItemBase extends FetchedItemBase {
+  releaseTimestamp: string;
+}
+
+export interface GithubDatasourceCache<StoredItem extends StoredItemBase> {
+  items: Record<string, StoredItem>;
+  cacheCreatedAt: string;
+  cacheUpdatedAt: string;
+}
+
+export interface GithubDatasourceCacheConfig<
+  FetchedItem extends FetchedItemBase,
+  StoredItem extends StoredItemBase
+> {
+  type: string;
+  query: string;
+  coerceFetched: (x: FetchedItem) => StoredItem;
+  isEquivalent: (x: StoredItem, y: StoredItem) => boolean;
+  coerceStored: (x: StoredItem) => Release;
+  softResetMinutes?: number;
+  hardResetDays?: number;
+  stabilityPeriodDays?: number;
+  maxPages?: number;
+}
+
+function isExpired(
+  now: DateTime,
+  date: string,
+  duration: DurationLike
+): boolean {
+  const then = DateTime.fromISO(date);
+  return now >= then.plus(duration);
+}
+
+export abstract class AbstractGithubDatasourceCache<
+  FetchedItem extends FetchedItemBase,
+  StoredItem extends StoredItemBase
+> {
+  constructor(
+    private http: GithubHttp,
+    private softResetMinutes = 30,
+    private hardResetDays = 7,
+    private stabilityPeriodDays = 31,
+    private perPage = 100,
+    private maxPages = 10
+  ) {}
+
+  abstract readonly cacheNs: string;
+  abstract readonly graphqlQuery: string;
+  abstract coerceFetched(fetchedItem: FetchedItem): StoredItem | null;
+  abstract isEquivalent(oldItem: StoredItem, newItem: StoredItem): boolean;
+  abstract coerceStored(storedItem: StoredItem): Release;
+
+  async getReleases(releasesConfig: GetReleasesConfig): Promise<Release[]> {
+    const { packageName, registryUrl } = releasesConfig;
+
+    const softReset: DurationLike = { minutes: this.softResetMinutes };
+    const hardReset: DurationLike = { days: this.hardResetDays };
+    const stabilityPeriod: DurationLike = { days: this.stabilityPeriodDays };
+
+    const now = DateTime.now();
+    let cache: GithubDatasourceCache<StoredItem> = {
+      items: {},
+      cacheCreatedAt: now.toISO(),
+      cacheUpdatedAt: now.minus(softReset).toISO(),
+    };
+
+    const baseUrl = getApiBaseUrl(registryUrl).replace('/v3/', '/');
+    const [owner, name] = packageName.split('/');
+    if (owner && name) {
+      const cacheKey = `${baseUrl}:${owner}/${name}`;
+      const cachedRes = await packageCache.get<
+        GithubDatasourceCache<StoredItem>
+      >(this.cacheNs, cacheKey);
+
+      let isCacheUpdated = false;
+
+      if (cachedRes && !isExpired(now, cachedRes.cacheCreatedAt, hardReset)) {
+        cache = cachedRes;
+      } else {
+        isCacheUpdated = true;
+      }
+
+      if (isExpired(now, cache.cacheUpdatedAt, softReset)) {
+        const variables: GithubQueryParams = {
+          owner,
+          name,
+          cursor: null,
+          count: this.perPage,
+        };
+
+        const checkedItems = new Set<string>();
+
+        try {
+          let pagesAllowed = this.maxPages;
+          let isIterating = true;
+          while (pagesAllowed > 0 && isIterating) {
+            const graphqlRes = await this.http.postJson<
+              GithubGraphqlResponse<QueryResponse<FetchedItem>>
+            >('/graphql', {
+              baseUrl,
+              body: { query: this.graphqlQuery, variables },
+            });
+            pagesAllowed -= 1;
+
+            const data = graphqlRes.body.data;
+            if (data) {
+              const {
+                nodes: fetchedItems,
+                pageInfo: { hasNextPage, endCursor },
+              } = data.repository.payload;
+
+              if (hasNextPage) {
+                variables.cursor = endCursor;
+              } else {
+                isIterating = false;
+              }
+
+              for (const item of fetchedItems) {
+                const newStoredItem = this.coerceFetched(item);
+                if (newStoredItem) {
+                  const { version, releaseTimestamp } = newStoredItem;
+                  checkedItems.add(version);
+
+                  const oldStoredItem = cache.items[version];
+                  if (
+                    !oldStoredItem ||
+                    !this.isEquivalent(oldStoredItem, newStoredItem)
+                  ) {
+                    cache.items[version] = newStoredItem;
+                    isCacheUpdated = true;
+                  } else if (
+                    isExpired(now, releaseTimestamp, stabilityPeriod)
+                  ) {
+                    isIterating = false;
+                    break;
+                  }
+                }
+              }
+            }
+          }
+        } catch (err) {
+          logger.debug(
+            { err },
+            `GitHub datasource: error fetching cacheable GraphQL data`
+          );
+        }
+
+        for (const [version, item] of Object.entries(cache.items)) {
+          if (
+            !isExpired(now, item.releaseTimestamp, stabilityPeriod) &&
+            !checkedItems.has(version)
+          ) {
+            delete cache.items[version];
+            isCacheUpdated = true;
+          }
+        }
+
+        if (isCacheUpdated) {
+          const expiry = DateTime.fromISO(cache.cacheCreatedAt).plus(hardReset);
+          const { minutes: ttlMinutes } = expiry
+            .diff(now, ['minutes'])
+            .toObject();
+          if (ttlMinutes && ttlMinutes > 0) {
+            cache.cacheUpdatedAt = now.toISO();
+            await packageCache.set(this.cacheNs, cacheKey, cache, ttlMinutes);
+          }
+        }
+      }
+    }
+
+    const storedItems = Object.values(cache.items);
+    return storedItems.map((item) => this.coerceStored(item));
+  }
+}

lib/modules/datasource/github-releases/cache.ts

@@ -0,0 +1,79 @@
+import type { GithubHttp } from '../../../util/http/github';
+import type { Release } from '../types';
+import {
+  AbstractGithubDatasourceCache,
+  FetchedItemBase,
+  StoredItemBase,
+} from './cache-base';
+
+export const query = `
+query ($owner: String!, $name: String!, $cursor: String, $count: Int!) {
+  repository(owner: $owner, name: $name) {
+    payload: releases(
+      after: $cursor
+      first: $count
+      orderBy: {field: CREATED_AT, direction: DESC}
+    ) {
+      nodes {
+        version: tagName
+        gitRef: tagName
+        releaseTimestamp: publishedAt
+        isDraft
+        isPrerelease
+        updatedAt
+      }
+      pageInfo {
+        hasNextPage
+        endCursor
+      }
+    }
+  }
+}
+`;
+
+interface FetchedRelease extends FetchedItemBase {
+  isDraft: boolean;
+  isPrerelease: boolean;
+  updatedAt: string;
+  releaseTimestamp: string;
+}
+
+interface StoredRelease extends StoredItemBase {
+  isStable?: boolean;
+  updatedAt: string;
+}
+
+export class CacheableGithubReleases extends AbstractGithubDatasourceCache<
+  FetchedRelease,
+  StoredRelease
+> {
+  cacheNs = 'github-datasource-graphql-releases';
+  graphqlQuery = query;
+
+  constructor(http: GithubHttp) {
+    super(http);
+  }
+
+  coerceFetched(item: FetchedRelease): StoredRelease {
+    const { version, releaseTimestamp, isDraft, isPrerelease, updatedAt } =
+      item;
+    const result: StoredRelease = { version, releaseTimestamp, updatedAt };
+    if (isPrerelease || isDraft) {
+      result.isStable = false;
+    }
+    return result;
+  }
+
+  coerceStored(item: StoredRelease): Release {
+    const { version, releaseTimestamp, isStable } = item;
+    const result: Release = { version, releaseTimestamp };
+    if (isStable !== undefined) {
+      result.isStable = isStable;
+    }
+    return result;
+  }
+
+  isEquivalent(oldItem: StoredRelease, newItem: StoredRelease): boolean {
+    return oldItem.releaseTimestamp === newItem.releaseTimestamp;
+  }
+}

lib/modules/datasource/github-releases/index.ts

@@ -5,6 +5,7 @@ import { GithubHttp } from '../../../util/http/github';
 import { newlineRegex, regEx } from '../../../util/regex';
 import { Datasource } from '../datasource';
 import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
+import { CacheableGithubReleases } from './cache';
 import { getApiBaseUrl, getSourceUrl } from './common';
 import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types';
 
@@ -27,9 +28,12 @@ export class GithubReleasesDatasource extends Datasource {
 
   override http: GithubHttp;
 
+  private releasesCache: CacheableGithubReleases;
+
   constructor(id = GithubReleasesDatasource.id) {
     super(id);
     this.http = new GithubHttp(id);
+    this.releasesCache = new CacheableGithubReleases(this.http);
   }
 
   async findDigestFile(
@@ -218,11 +222,6 @@ export class GithubReleasesDatasource extends Datasource {
     return newDigest;
   }
 
-  @cache({
-    namespace: 'datasource-github-releases',
-    key: ({ packageName: repo, registryUrl }: GetReleasesConfig) =>
-      `${registryUrl}:${repo}:tags`,
-  })
   /**
    * github.getReleases
    *
@@ -233,27 +232,13 @@ export class GithubReleasesDatasource extends Datasource {
    *  - Sanitize the versions if desired (e.g. strip out leading 'v')
    *  - Return a dependency object containing sourceUrl string and releases array
    */
-  async getReleases({
-    packageName: repo,
-    registryUrl,
-  }: GetReleasesConfig): Promise<ReleaseResult | null> {
-    const apiBaseUrl = getApiBaseUrl(registryUrl);
-    const url = `${apiBaseUrl}repos/${repo}/releases?per_page=100`;
-    const res = await this.http.getJson<GithubRelease[]>(url, {
-      paginate: true,
-    });
-    const githubReleases = res.body;
-    const dependency: ReleaseResult = {
-      sourceUrl: getSourceUrl(repo, registryUrl),
-      releases: githubReleases
-        .filter(({ draft }) => draft !== true)
-        .map(({ tag_name, published_at, prerelease }) => ({
-          version: tag_name,
-          gitRef: tag_name,
-          releaseTimestamp: published_at,
-          isStable: prerelease ? false : undefined,
-        })),
-    };
-    return dependency;
+  async getReleases(config: GetReleasesConfig): Promise<ReleaseResult | null> {
+    const releases = await this.releasesCache.getReleases(config);
+    return releases.length
+      ? {
+          sourceUrl: getSourceUrl(config.packageName, config.registryUrl),
+          releases,
+        }
+      : null;
   }
 }