From 2e957baed962d65cb8e40136edc142af6014ad95 Mon Sep 17 00:00:00 2001 From: Sergei Zharinov Date: Fri, 3 Jun 2022 12:27:26 +0300 Subject: [PATCH] feat(github): long-term datasource caching (#15653) --- .../__snapshots__/index.spec.ts.snap | 64 ---- .../github-releases/cache/cache-base.spec.ts | 262 +++++++++++++++ .../github-releases/cache/cache-base.ts | 301 ++++++++++++++++++ .../github-releases/cache/index.spec.ts | 43 +++ .../datasource/github-releases/cache/index.ts | 93 ++++++ .../datasource/github-releases/cache/types.ts | 100 ++++++ .../datasource/github-releases/common.ts | 5 +- .../datasource/github-releases/index.spec.ts | 71 ++--- .../datasource/github-releases/index.ts | 55 ++-- .../__snapshots__/index.spec.ts.snap | 37 --- .../datasource/github-tags/cache.spec.ts | 51 +++ lib/modules/datasource/github-tags/cache.ts | 88 +++++ .../datasource/github-tags/index.spec.ts | 68 ++-- lib/modules/datasource/github-tags/index.ts | 44 +-- .../releases-direct.spec.ts.snap | 32 -- .../datasource/go/releases-direct.spec.ts | 127 ++++---- .../datasource/go/releases-goproxy.spec.ts | 38 ++- lib/util/http/github.ts | 2 +- .../repository/process/lookup/index.spec.ts | 74 ++--- .../update/pr/changelog/github.spec.ts | 22 +- .../update/pr/changelog/github/index.ts | 40 +-- .../update/pr/changelog/index.spec.ts | 61 ++-- .../update/pr/changelog/release-notes.spec.ts | 154 +++++---- 23 files changed, 1318 insertions(+), 514 deletions(-) delete mode 100644 lib/modules/datasource/github-releases/__snapshots__/index.spec.ts.snap create mode 100644 lib/modules/datasource/github-releases/cache/cache-base.spec.ts create mode 100644 lib/modules/datasource/github-releases/cache/cache-base.ts create mode 100644 lib/modules/datasource/github-releases/cache/index.spec.ts create mode 100644 lib/modules/datasource/github-releases/cache/index.ts create mode 100644 lib/modules/datasource/github-releases/cache/types.ts delete mode 100644 lib/modules/datasource/github-tags/__snapshots__/index.spec.ts.snap create mode 100644 lib/modules/datasource/github-tags/cache.spec.ts create mode 100644 lib/modules/datasource/github-tags/cache.ts diff --git a/lib/modules/datasource/github-releases/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/github-releases/__snapshots__/index.spec.ts.snap deleted file mode 100644 index 4bf63b80cdd5ed..00000000000000 --- a/lib/modules/datasource/github-releases/__snapshots__/index.spec.ts.snap +++ /dev/null @@ -1,64 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`modules/datasource/github-releases/index getReleases returns releases 1`] = ` -Object { - "registryUrl": "https://github.com", - "releases": Array [ - Object { - "gitRef": "1.0.0", - "releaseTimestamp": "2020-03-09T11:00:00.000Z", - "version": "1.0.0", - }, - Object { - "gitRef": "v1.1.0", - "releaseTimestamp": "2020-03-09T10:00:00.000Z", - "version": "v1.1.0", - }, - Object { - "gitRef": "2.0.0", - "isStable": false, - "releaseTimestamp": "2020-04-09T10:00:00.000Z", - "version": "2.0.0", - }, - ], - "sourceUrl": "https://github.com/some/dep", -} -`; - -exports[`modules/datasource/github-releases/index getReleases supports ghe 1`] = ` -Object { - "releases": Array [ - Object { - "gitRef": "a", - "isStable": undefined, - "releaseTimestamp": "2020-03-09T13:00:00Z", - "version": "a", - }, - Object { - "gitRef": "v", - "isStable": undefined, - "releaseTimestamp": "2020-03-09T12:00:00Z", - "version": "v", - }, - Object { - "gitRef": "1.0.0", - "isStable": undefined, - "releaseTimestamp": "2020-03-09T11:00:00Z", - "version": "1.0.0", - }, - Object { - "gitRef": "v1.1.0", - "isStable": undefined, - "releaseTimestamp": "2020-03-09T10:00:00Z", - "version": "v1.1.0", - }, - Object { - "gitRef": "2.0.0", - "isStable": false, - "releaseTimestamp": "2020-04-09T10:00:00Z", - "version": "2.0.0", - }, - ], - "sourceUrl": "https://git.enterprise.com/some/dep", -} -`; diff --git a/lib/modules/datasource/github-releases/cache/cache-base.spec.ts b/lib/modules/datasource/github-releases/cache/cache-base.spec.ts new file mode 100644 index 00000000000000..c36ffccd4002b3 --- /dev/null +++ b/lib/modules/datasource/github-releases/cache/cache-base.spec.ts @@ -0,0 +1,262 @@ +import { DateTime } from 'luxon'; +import { mocked } from '../../../../../test/util'; +import * as _packageCache from '../../../../util/cache/package'; +import { + GithubGraphqlResponse, + GithubHttp, +} from '../../../../util/http/github'; +import { AbstractGithubDatasourceCache } from './cache-base'; +import type { QueryResponse, StoredItemBase } from './types'; + +jest.mock('../../../../util/cache/package'); +const packageCache = mocked(_packageCache); + +interface FetchedItem { + name: string; + createdAt: string; + foo: string; +} + +interface StoredItem extends StoredItemBase { + bar: string; +} + +type GraphqlDataResponse = { + statusCode: 200; + headers: Record; + body: GithubGraphqlResponse>; +}; + +type GraphqlResponse = GraphqlDataResponse | Error; + +class TestCache extends AbstractGithubDatasourceCache { + cacheNs = 'test-cache'; + graphqlQuery = `query { ... }`; + + coerceFetched({ + name: version, + createdAt: releaseTimestamp, + foo: bar, + }: FetchedItem): StoredItem | null { + return version === 'invalid' ? null : { version, releaseTimestamp, bar }; + } + + isEquivalent({ bar: x }: StoredItem, { bar: y }: StoredItem): boolean { + return x === y; + } +} + +function resp(items: FetchedItem[], hasNextPage = false): GraphqlDataResponse { + return { + statusCode: 200, + headers: {}, + body: { + data: { + repository: { + payload: { + nodes: items, + pageInfo: { + hasNextPage, + endCursor: 'abc', + }, + }, + }, + }, + }, + }; +} + +const sortItems = (items: StoredItem[]) => + items.sort(({ releaseTimestamp: x }, { releaseTimestamp: y }) => + x.localeCompare(y) + ); + +describe('modules/datasource/github-releases/cache/cache-base', () => { + const http = new GithubHttp(); + const httpPostJson = jest.spyOn(GithubHttp.prototype, 'postJson'); + + const now = DateTime.local(2022, 6, 15, 18, 30, 30); + const t1 = now.minus({ days: 3 }).toISO(); + const t2 = now.minus({ days: 2 }).toISO(); + const t3 = now.minus({ days: 1 }).toISO(); + + let responses: GraphqlResponse[] = []; + + beforeEach(() => { + responses = []; + jest.resetAllMocks(); + jest.spyOn(DateTime, 'now').mockReturnValue(now); + httpPostJson.mockImplementation(() => { + const resp = responses.shift(); + return resp instanceof Error + ? Promise.reject(resp) + : Promise.resolve(resp); + }); + }); + + it('performs pre-fetch', async () => { + responses = [ + resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true), + resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true), + resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), + ]; + const cache = new TestCache(http, { resetDeltaMinutes: 0 }); + + const res = await cache.getItems({ packageName: 'foo/bar' }); + + expect(sortItems(res)).toMatchObject([ + { version: 'v1', bar: 'aaa' }, + { version: 'v2', bar: 'bbb' }, + { version: 'v3', bar: 'ccc' }, + ]); + expect(packageCache.set).toHaveBeenCalledWith( + 'test-cache', + 'https://api.github.com/:foo:bar', + { + createdAt: now.toISO(), + updatedAt: now.toISO(), + items: { + v1: { bar: 'aaa', releaseTimestamp: t1, version: 'v1' }, + v2: { bar: 'bbb', releaseTimestamp: t2, version: 'v2' }, + v3: { bar: 'ccc', releaseTimestamp: t3, version: 'v3' }, + }, + }, + 7 * 24 * 60 + ); + }); + + it('filters out items being coerced to null', async () => { + responses = [ + resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true), + resp([{ name: 'invalid', createdAt: t3, foo: 'xxx' }], true), + resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true), + resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), + ]; + const cache = new TestCache(http, { resetDeltaMinutes: 0 }); + + const res = await cache.getItems({ packageName: 'foo/bar' }); + + expect(sortItems(res)).toMatchObject([ + { version: 'v1' }, + { version: 'v2' }, + { version: 'v3' }, + ]); + }); + + it('updates items', async () => { + packageCache.get.mockResolvedValueOnce({ + items: { + v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, + v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, + v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, + }, + createdAt: t3, + updatedAt: t3, + }); + + responses = [ + resp([{ name: 'v3', createdAt: t3, foo: 'xxx' }], true), + resp([{ name: 'v2', createdAt: t2, foo: 'bbb' }], true), + resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), + ]; + const cache = new TestCache(http, { resetDeltaMinutes: 0 }); + + const res = await cache.getItems({ packageName: 'foo/bar' }); + + expect(sortItems(res)).toMatchObject([ + { version: 'v1', bar: 'aaa' }, + { version: 'v2', bar: 'bbb' }, + { version: 'v3', bar: 'xxx' }, + ]); + expect(packageCache.set).toHaveBeenCalledWith( + 'test-cache', + 'https://api.github.com/:foo:bar', + { + createdAt: t3, + updatedAt: now.toISO(), + items: { + v1: { bar: 'aaa', releaseTimestamp: t1, version: 'v1' }, + v2: { bar: 'bbb', releaseTimestamp: t2, version: 'v2' }, + v3: { bar: 'xxx', releaseTimestamp: t3, version: 'v3' }, + }, + }, + 6 * 24 * 60 + ); + }); + + it('stops updating once stability period have passed', async () => { + packageCache.get.mockResolvedValueOnce({ + items: { + v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, + v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, + v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, + }, + createdAt: t3, + updatedAt: t3, + }); + responses = [ + resp([{ name: 'v3', createdAt: t3, foo: 'zzz' }], true), + resp([{ name: 'v2', createdAt: t2, foo: 'yyy' }], true), + resp([{ name: 'v1', createdAt: t1, foo: 'xxx' }]), + ]; + const cache = new TestCache(http, { unstableDays: 1.5 }); + + const res = await cache.getItems({ packageName: 'foo/bar' }); + + expect(sortItems(res)).toMatchObject([ + { version: 'v1', bar: 'aaa' }, + { version: 'v2', bar: 'bbb' }, + { version: 'v3', bar: 'zzz' }, + ]); + }); + + it('removes deleted items from cache', async () => { + packageCache.get.mockResolvedValueOnce({ + items: { + v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, + v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, + v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, + }, + createdAt: t3, + updatedAt: t3, + }); + responses = [ + resp([{ name: 'v3', createdAt: t3, foo: 'ccc' }], true), + resp([{ name: 'v1', createdAt: t1, foo: 'aaa' }]), + ]; + const cache = new TestCache(http, { resetDeltaMinutes: 0 }); + + const res = await cache.getItems({ packageName: 'foo/bar' }); + + expect(sortItems(res)).toMatchObject([ + { version: 'v1', bar: 'aaa' }, + { version: 'v3', bar: 'ccc' }, + ]); + }); + + it('returns cached values on server errors', async () => { + packageCache.get.mockResolvedValueOnce({ + items: { + v1: { version: 'v1', releaseTimestamp: t1, bar: 'aaa' }, + v2: { version: 'v2', releaseTimestamp: t2, bar: 'bbb' }, + v3: { version: 'v3', releaseTimestamp: t3, bar: 'ccc' }, + }, + createdAt: t3, + updatedAt: t3, + }); + responses = [ + resp([{ name: 'v3', createdAt: t3, foo: 'zzz' }], true), + new Error('Unknown error'), + resp([{ name: 'v1', createdAt: t1, foo: 'xxx' }]), + ]; + const cache = new TestCache(http, { resetDeltaMinutes: 0 }); + + const res = await cache.getItems({ packageName: 'foo/bar' }); + + expect(sortItems(res)).toMatchObject([ + { version: 'v1', bar: 'aaa' }, + { version: 'v2', bar: 'bbb' }, + { version: 'v3', bar: 'ccc' }, + ]); + }); +}); diff --git a/lib/modules/datasource/github-releases/cache/cache-base.ts b/lib/modules/datasource/github-releases/cache/cache-base.ts new file mode 100644 index 00000000000000..8e8ee6ae172e7d --- /dev/null +++ b/lib/modules/datasource/github-releases/cache/cache-base.ts @@ -0,0 +1,301 @@ +import { DateTime, DurationLikeObject } from 'luxon'; +import { logger } from '../../../../logger'; +import * as packageCache from '../../../../util/cache/package'; +import type { + GithubGraphqlResponse, + GithubHttp, +} from '../../../../util/http/github'; +import type { GetReleasesConfig } from '../../types'; +import { getApiBaseUrl } from '../common'; +import type { + CacheOptions, + GithubDatasourceCache, + GithubQueryParams, + QueryResponse, + StoredItemBase, +} from './types'; + +/** + * The options that are meant to be used in production. + */ +const cacheDefaults: Required = { + /** + * How many minutes to wait until next cache update + */ + updateAfterMinutes: 30, + + /** + * How many days to wait until full cache reset (for single package). + */ + resetAfterDays: 7, + + /** + * Delays cache reset by some random amount of minutes, + * in order to stabilize load during mass cache reset. + */ + resetDeltaMinutes: 3 * 60, + + /** + * How many days ago the package should be published to be considered as stable. + * Since this period is expired, it won't be refreshed via soft updates anymore. + */ + unstableDays: 30, + + /** + * How many items per page to obtain per page during initial fetch (i.e. pre-fetch) + */ + itemsPerPrefetchPage: 100, + + /** + * How many pages to fetch (at most) during the initial fetch (i.e. pre-fetch) + */ + maxPrefetchPages: 100, + + /** + * How many items per page to obtain per page during the soft update + */ + itemsPerUpdatePage: 100, + + /** + * How many pages to fetch (at most) during the soft update + */ + maxUpdatePages: 100, +}; + +/** + * Tells whether the time `duration` is expired starting + * from the `date` (ISO date format) at the moment of `now`. + */ +function isExpired( + now: DateTime, + date: string, + duration: DurationLikeObject +): boolean { + const then = DateTime.fromISO(date); + const expiry = then.plus(duration); + return now >= expiry; +} + +export abstract class AbstractGithubDatasourceCache< + StoredItem extends StoredItemBase, + FetchedItem = unknown +> { + private updateDuration: DurationLikeObject; + private resetDuration: DurationLikeObject; + private stabilityDuration: DurationLikeObject; + + private maxPrefetchPages: number; + private itemsPerPrefetchPage: number; + + private maxUpdatePages: number; + private itemsPerUpdatePage: number; + + private resetDeltaMinutes: number; + + constructor(private http: GithubHttp, opts: CacheOptions = {}) { + const { + updateAfterMinutes, + resetAfterDays, + unstableDays, + maxPrefetchPages, + itemsPerPrefetchPage, + maxUpdatePages, + itemsPerUpdatePage, + resetDeltaMinutes, + } = { + ...cacheDefaults, + ...opts, + }; + + this.updateDuration = { minutes: updateAfterMinutes }; + this.resetDuration = { days: resetAfterDays }; + this.stabilityDuration = { days: unstableDays }; + + this.maxPrefetchPages = maxPrefetchPages; + this.itemsPerPrefetchPage = itemsPerPrefetchPage; + this.maxUpdatePages = maxUpdatePages; + this.itemsPerUpdatePage = itemsPerUpdatePage; + + this.resetDeltaMinutes = resetDeltaMinutes; + } + + /** + * The key at which data is stored in the package cache. + */ + abstract readonly cacheNs: string; + + /** + * The query string. + * For parameters, see `GithubQueryParams`. + */ + abstract readonly graphqlQuery: string; + + /** + * Transform `fetchedItem` for storing in the package cache. + * @param fetchedItem Node obtained from GraphQL response + */ + abstract coerceFetched(fetchedItem: FetchedItem): StoredItem | null; + + /** + * Pre-fetch, update, or just return the package cache items. + */ + async getItems(releasesConfig: GetReleasesConfig): Promise { + const { packageName, registryUrl } = releasesConfig; + + // The time meant to be used across the function + const now = DateTime.now(); + + // Initialize items and timestamps for the new cache + let cacheItems: Record = {}; + + // Add random minutes to the creation date in order to + // provide back-off time during mass cache invalidation. + const randomDelta = this.getRandomDeltaMinutes(); + let cacheCreatedAt = now.plus(randomDelta).toISO(); + + // We have to initialize `updatedAt` value as already expired, + // so that soft update mechanics is immediately starting. + let cacheUpdatedAt = now.minus(this.updateDuration).toISO(); + + const baseUrl = getApiBaseUrl(registryUrl).replace('/v3/', '/'); // Replace for GHE + + const [owner, name] = packageName.split('/'); + if (owner && name) { + const cacheKey = `${baseUrl}:${owner}:${name}`; + const cache = await packageCache.get>( + this.cacheNs, + cacheKey + ); + + const cacheDoesExist = + cache && !isExpired(now, cache.createdAt, this.resetDuration); + if (cacheDoesExist) { + // Keeping the the original `cache` value intact + // in order to be used in exception handler + cacheItems = { ...cache.items }; + cacheCreatedAt = cache.createdAt; + cacheUpdatedAt = cache.updatedAt; + } + + try { + if (isExpired(now, cacheUpdatedAt, this.updateDuration)) { + const variables: GithubQueryParams = { + owner, + name, + cursor: null, + count: cacheDoesExist + ? this.itemsPerUpdatePage + : this.itemsPerPrefetchPage, + }; + + // Collect version values to determine deleted items + const checkedVersions = new Set(); + + // Page-by-page update loop + let pagesRemained = cacheDoesExist + ? this.maxUpdatePages + : this.maxPrefetchPages; + let stopIteration = false; + while (pagesRemained > 0 && !stopIteration) { + const graphqlRes = await this.http.postJson< + GithubGraphqlResponse> + >('/graphql', { + baseUrl, + body: { query: this.graphqlQuery, variables }, + }); + pagesRemained -= 1; + + const data = graphqlRes.body.data; + if (data) { + const { + nodes: fetchedItems, + pageInfo: { hasNextPage, endCursor }, + } = data.repository.payload; + + if (hasNextPage) { + variables.cursor = endCursor; + } else { + stopIteration = true; + } + + for (const item of fetchedItems) { + const newStoredItem = this.coerceFetched(item); + if (newStoredItem) { + const { version } = newStoredItem; + + // Stop earlier if the stored item have reached stability, + // which means `unstableDays` period have passed + const oldStoredItem = cacheItems[version]; + if ( + oldStoredItem && + isExpired( + now, + oldStoredItem.releaseTimestamp, + this.stabilityDuration + ) + ) { + stopIteration = true; + break; + } + + cacheItems[version] = newStoredItem; + checkedVersions.add(version); + } + } + } + } + + // Detect removed items + for (const [version, item] of Object.entries(cacheItems)) { + if ( + !isExpired(now, item.releaseTimestamp, this.stabilityDuration) && + !checkedVersions.has(version) + ) { + delete cacheItems[version]; + } + } + + // Store cache + const expiry = DateTime.fromISO(cacheCreatedAt).plus( + this.resetDuration + ); + const { minutes: ttlMinutes } = expiry + .diff(now, ['minutes']) + .toObject(); + if (ttlMinutes && ttlMinutes > 0) { + const cacheValue: GithubDatasourceCache = { + items: cacheItems, + createdAt: cacheCreatedAt, + updatedAt: now.toISO(), + }; + await packageCache.set( + this.cacheNs, + cacheKey, + cacheValue, + ttlMinutes + ); + } + } + } catch (err) { + logger.debug( + { err }, + `GitHub datasource: error fetching cacheable GraphQL data` + ); + + // On errors, return previous value (if valid) + if (cacheDoesExist) { + const cachedItems = Object.values(cache.items); + return cachedItems; + } + } + } + + const items = Object.values(cacheItems); + return items; + } + + getRandomDeltaMinutes(): number { + const rnd = Math.random(); + return Math.floor(rnd * this.resetDeltaMinutes); + } +} diff --git a/lib/modules/datasource/github-releases/cache/index.spec.ts b/lib/modules/datasource/github-releases/cache/index.spec.ts new file mode 100644 index 00000000000000..9839f49432109f --- /dev/null +++ b/lib/modules/datasource/github-releases/cache/index.spec.ts @@ -0,0 +1,43 @@ +import { GithubHttp } from '../../../../util/http/github'; +import { CacheableGithubReleases, FetchedRelease } from '.'; + +describe('modules/datasource/github-releases/cache/index', () => { + const http = new GithubHttp(); + const cache = new CacheableGithubReleases(http, { resetDeltaMinutes: 0 }); + + const fetchedItem: FetchedRelease = { + version: '1.2.3', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + isDraft: false, + isPrerelease: false, + url: 'https://example.com/', + id: 123, + name: 'Some name', + description: 'Some description', + }; + + describe('coerceFetched', () => { + it('transforms GraphQL item', () => { + expect(cache.coerceFetched(fetchedItem)).toEqual({ + description: 'Some description', + id: 123, + name: 'Some name', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + url: 'https://example.com/', + version: '1.2.3', + }); + }); + + it('marks pre-release as unstable', () => { + expect( + cache.coerceFetched({ ...fetchedItem, isPrerelease: true }) + ).toMatchObject({ + isStable: false, + }); + }); + + it('filters out drafts', () => { + expect(cache.coerceFetched({ ...fetchedItem, isDraft: true })).toBeNull(); + }); + }); +}); diff --git a/lib/modules/datasource/github-releases/cache/index.ts b/lib/modules/datasource/github-releases/cache/index.ts new file mode 100644 index 00000000000000..5f7d35cfb6f8e9 --- /dev/null +++ b/lib/modules/datasource/github-releases/cache/index.ts @@ -0,0 +1,93 @@ +import type { GithubHttp } from '../../../../util/http/github'; +import { AbstractGithubDatasourceCache } from './cache-base'; +import type { CacheOptions, StoredItemBase } from './types'; + +export const query = ` +query ($owner: String!, $name: String!, $cursor: String, $count: Int!) { + repository(owner: $owner, name: $name) { + payload: releases( + first: $count + after: $cursor + orderBy: {field: CREATED_AT, direction: DESC} + ) { + nodes { + version: tagName + releaseTimestamp: publishedAt + isDraft + isPrerelease + url + id: databaseId + name + description + } + pageInfo { + hasNextPage + endCursor + } + } + } +} +`; + +export interface FetchedRelease { + version: string; + releaseTimestamp: string; + isDraft: boolean; + isPrerelease: boolean; + url: string; + id: number; + name: string; + description: string; +} + +export interface StoredRelease extends StoredItemBase { + isStable?: boolean; + url: string; + id: number; + name: string; + description: string; +} + +export class CacheableGithubReleases extends AbstractGithubDatasourceCache< + StoredRelease, + FetchedRelease +> { + cacheNs = 'github-datasource-graphql-releases'; + graphqlQuery = query; + + constructor(http: GithubHttp, opts: CacheOptions = {}) { + super(http, opts); + } + + coerceFetched(item: FetchedRelease): StoredRelease | null { + const { + version, + releaseTimestamp, + isDraft, + isPrerelease, + url, + id, + name, + description, + } = item; + + if (isDraft) { + return null; + } + + const result: StoredRelease = { + version, + releaseTimestamp, + url, + id, + name, + description, + }; + + if (isPrerelease) { + result.isStable = false; + } + + return result; + } +} diff --git a/lib/modules/datasource/github-releases/cache/types.ts b/lib/modules/datasource/github-releases/cache/types.ts new file mode 100644 index 00000000000000..898aa724e62f1d --- /dev/null +++ b/lib/modules/datasource/github-releases/cache/types.ts @@ -0,0 +1,100 @@ +/** + * Every `AbstractGithubDatasourceCache` implementation + * should have `graphqlQuery` that uses parameters + * defined this interface. + */ +export interface GithubQueryParams { + owner: string; + name: string; + cursor: string | null; + count: number; +} + +/** + * Every `AbstractGithubDatasourceCache` implementation + * should have `graphqlQuery` that resembles the structure + * of this interface. + */ +export interface QueryResponse { + repository: { + payload: { + nodes: T[]; + pageInfo: { + hasNextPage: boolean; + endCursor: string; + }; + }; + }; +} + +/** + * Base interface meant to be extended by all implementations. + * Must have `version` and `releaseTimestamp` fields. + */ +export interface StoredItemBase { + /** The values of `version` field meant to be unique. */ + version: string; + + /** The `releaseTimestamp` field meant to be ISO-encoded date. */ + releaseTimestamp: string; +} + +/** + * The data structure stored in the package cache. + */ +export interface GithubDatasourceCache { + items: Record; + + /** Cache full reset decision is based on `createdAt` value. */ + createdAt: string; + + /** Cache soft updates are performed depending on `updatedAt` value. */ + updatedAt: string; +} + +/** + * The configuration for cache. + */ +export interface CacheOptions { + /** + * How many minutes to wait until next cache update + */ + updateAfterMinutes?: number; + + /** + * How many days to wait until full cache reset (for single package). + */ + resetAfterDays?: number; + + /** + * Delays cache reset by some random amount of minutes, + * in order to stabilize load during mass cache reset. + */ + resetDeltaMinutes?: number; + + /** + * How many days ago the package should be published to be considered as stable. + * Since this period is expired, it won't be refreshed via soft updates anymore. + */ + unstableDays?: number; + + /** + * How many items per page to obtain per page during initial fetch (i.e. pre-fetch) + */ + itemsPerPrefetchPage?: number; + + /** + * How many pages to fetch (at most) during the initial fetch (i.e. pre-fetch) + */ + maxPrefetchPages?: number; + + /** + * How many items per page to obtain per page during the soft update + */ + itemsPerUpdatePage?: number; + + /** + * How many pages to fetch (at most) during the soft update + */ + maxUpdatePages?: number; +} diff --git a/lib/modules/datasource/github-releases/common.ts b/lib/modules/datasource/github-releases/common.ts index b0164ca1e1cc74..638ebdeff5f7a1 100644 --- a/lib/modules/datasource/github-releases/common.ts +++ b/lib/modules/datasource/github-releases/common.ts @@ -1,6 +1,7 @@ import { ensureTrailingSlash } from '../../../util/url'; const defaultSourceUrlBase = 'https://github.com/'; +const defaultApiBaseUrl = 'https://api.github.com/'; export function getSourceUrlBase(registryUrl: string | undefined): string { // default to GitHub.com if no GHE host is specified. @@ -9,8 +10,8 @@ export function getSourceUrlBase(registryUrl: string | undefined): string { export function getApiBaseUrl(registryUrl: string | undefined): string { const sourceUrlBase = getSourceUrlBase(registryUrl); - return sourceUrlBase === defaultSourceUrlBase - ? `https://api.github.com/` + return [defaultSourceUrlBase, defaultApiBaseUrl].includes(sourceUrlBase) + ? defaultApiBaseUrl : `${sourceUrlBase}api/v3/`; } diff --git a/lib/modules/datasource/github-releases/index.spec.ts b/lib/modules/datasource/github-releases/index.spec.ts index 2c0908962dcf06..dcfdb54654462e 100644 --- a/lib/modules/datasource/github-releases/index.spec.ts +++ b/lib/modules/datasource/github-releases/index.spec.ts @@ -1,6 +1,6 @@ import { getDigest, getPkgReleases } from '..'; -import * as httpMock from '../../../../test/http-mock'; import * as _hostRules from '../../../util/host-rules'; +import { CacheableGithubReleases } from './cache'; import { GitHubReleaseMocker } from './test'; import { GithubReleasesDatasource } from '.'; @@ -8,25 +8,15 @@ jest.mock('../../../util/host-rules'); const hostRules: any = _hostRules; const githubApiHost = 'https://api.github.com'; -const githubEnterpriseApiHost = 'https://git.enterprise.com'; - -const responseBody = [ - { tag_name: 'a', published_at: '2020-03-09T13:00:00Z' }, - { tag_name: 'v', published_at: '2020-03-09T12:00:00Z' }, - { tag_name: '1.0.0', published_at: '2020-03-09T11:00:00Z' }, - { tag_name: 'v1.1.0', draft: false, published_at: '2020-03-09T10:00:00Z' }, - { tag_name: '1.2.0', draft: true, published_at: '2020-03-09T10:00:00Z' }, - { - tag_name: '2.0.0', - published_at: '2020-04-09T10:00:00Z', - prerelease: true, - }, -]; describe('modules/datasource/github-releases/index', () => { - const githubReleases = new GithubReleasesDatasource(); + const cacheGetItems = jest.spyOn( + CacheableGithubReleases.prototype, + 'getItems' + ); beforeEach(() => { + jest.resetAllMocks(); hostRules.hosts.mockReturnValue([]); hostRules.find.mockReturnValue({ token: 'some-token', @@ -35,39 +25,36 @@ describe('modules/datasource/github-releases/index', () => { describe('getReleases', () => { it('returns releases', async () => { - httpMock - .scope(githubApiHost) - .get('/repos/some/dep/releases?per_page=100') - .reply(200, responseBody); + cacheGetItems.mockResolvedValueOnce([ + { version: 'a', releaseTimestamp: '2020-03-09T13:00:00Z' }, + { version: 'v', releaseTimestamp: '2020-03-09T12:00:00Z' }, + { version: '1.0.0', releaseTimestamp: '2020-03-09T11:00:00Z' }, + { version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00Z' }, + { + version: '2.0.0', + releaseTimestamp: '2020-04-09T10:00:00Z', + isStable: false, + }, + ] as never); const res = await getPkgReleases({ datasource: GithubReleasesDatasource.id, depName: 'some/dep', }); - expect(res).toMatchSnapshot(); - expect(res.releases).toHaveLength(3); - expect( - res.releases.find((release) => release.version === 'v1.1.0') - ).toBeDefined(); - expect( - res.releases.find((release) => release.version === '1.2.0') - ).toBeUndefined(); - expect( - res.releases.find((release) => release.version === '2.0.0').isStable - ).toBeFalse(); - }); - it('supports ghe', async () => { - const packageName = 'some/dep'; - httpMock - .scope(githubEnterpriseApiHost) - .get(`/api/v3/repos/${packageName}/releases?per_page=100`) - .reply(200, responseBody); - const res = await githubReleases.getReleases({ - registryUrl: 'https://git.enterprise.com', - packageName, + expect(res).toMatchObject({ + registryUrl: 'https://github.com', + releases: [ + { releaseTimestamp: '2020-03-09T11:00:00.000Z', version: '1.0.0' }, + { version: 'v1.1.0', releaseTimestamp: '2020-03-09T10:00:00.000Z' }, + { + version: '2.0.0', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + isStable: false, + }, + ], + sourceUrl: 'https://github.com/some/dep', }); - expect(res).toMatchSnapshot(); }); }); diff --git a/lib/modules/datasource/github-releases/index.ts b/lib/modules/datasource/github-releases/index.ts index 63f8db6ea1f186..c28d774e083114 100644 --- a/lib/modules/datasource/github-releases/index.ts +++ b/lib/modules/datasource/github-releases/index.ts @@ -4,7 +4,13 @@ import { cache } from '../../../util/cache/package/decorator'; import { GithubHttp } from '../../../util/http/github'; import { newlineRegex, regEx } from '../../../util/regex'; import { Datasource } from '../datasource'; -import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; +import type { + DigestConfig, + GetReleasesConfig, + Release, + ReleaseResult, +} from '../types'; +import { CacheableGithubReleases } from './cache'; import { getApiBaseUrl, getSourceUrl } from './common'; import type { DigestAsset, GithubRelease, GithubReleaseAsset } from './types'; @@ -27,9 +33,12 @@ export class GithubReleasesDatasource extends Datasource { override http: GithubHttp; + private releasesCache: CacheableGithubReleases; + constructor(id = GithubReleasesDatasource.id) { super(id); this.http = new GithubHttp(id); + this.releasesCache = new CacheableGithubReleases(this.http); } async findDigestFile( @@ -218,11 +227,6 @@ export class GithubReleasesDatasource extends Datasource { return newDigest; } - @cache({ - namespace: 'datasource-github-releases', - key: ({ packageName: repo, registryUrl }: GetReleasesConfig) => - `${registryUrl}:${repo}:tags`, - }) /** * github.getReleases * @@ -233,27 +237,22 @@ export class GithubReleasesDatasource extends Datasource { * - Sanitize the versions if desired (e.g. strip out leading 'v') * - Return a dependency object containing sourceUrl string and releases array */ - async getReleases({ - packageName: repo, - registryUrl, - }: GetReleasesConfig): Promise { - const apiBaseUrl = getApiBaseUrl(registryUrl); - const url = `${apiBaseUrl}repos/${repo}/releases?per_page=100`; - const res = await this.http.getJson(url, { - paginate: true, - }); - const githubReleases = res.body; - const dependency: ReleaseResult = { - sourceUrl: getSourceUrl(repo, registryUrl), - releases: githubReleases - .filter(({ draft }) => draft !== true) - .map(({ tag_name, published_at, prerelease }) => ({ - version: tag_name, - gitRef: tag_name, - releaseTimestamp: published_at, - isStable: prerelease ? false : undefined, - })), - }; - return dependency; + async getReleases(config: GetReleasesConfig): Promise { + let result: ReleaseResult | null = null; + const releases = await this.releasesCache.getItems(config); + if (releases.length) { + result = { + sourceUrl: getSourceUrl(config.packageName, config.registryUrl), + releases: releases.map((item) => { + const { version, releaseTimestamp, isStable } = item; + const result: Release = { version, releaseTimestamp }; + if (isStable !== undefined) { + result.isStable = isStable; + } + return result; + }), + }; + } + return result; } } diff --git a/lib/modules/datasource/github-tags/__snapshots__/index.spec.ts.snap b/lib/modules/datasource/github-tags/__snapshots__/index.spec.ts.snap deleted file mode 100644 index 2fad1922db34a3..00000000000000 --- a/lib/modules/datasource/github-tags/__snapshots__/index.spec.ts.snap +++ /dev/null @@ -1,37 +0,0 @@ -// Jest Snapshot v1, https://goo.gl/fbAQLP - -exports[`modules/datasource/github-tags/index getReleases returns tags 1`] = ` -Object { - "registryUrl": "https://github.com", - "releases": Array [ - Object { - "gitRef": "v1.0.0", - "releaseTimestamp": "1970-01-01T00:00:00.000Z", - "version": "v1.0.0", - }, - Object { - "gitRef": "v1.1.0", - "isStable": false, - "releaseTimestamp": "1970-01-01T00:00:00.001Z", - "version": "v1.1.0", - }, - ], - "sourceUrl": "https://github.com/some/dep2", -} -`; - -exports[`modules/datasource/github-tags/index getReleases supports ghe 1`] = ` -Object { - "releases": Array [ - Object { - "gitRef": "v1.0.0", - "version": "v1.0.0", - }, - Object { - "gitRef": "v1.1.0", - "version": "v1.1.0", - }, - ], - "sourceUrl": "https://git.enterprise.com/some/dep2", -} -`; diff --git a/lib/modules/datasource/github-tags/cache.spec.ts b/lib/modules/datasource/github-tags/cache.spec.ts new file mode 100644 index 00000000000000..3d8f02ff059774 --- /dev/null +++ b/lib/modules/datasource/github-tags/cache.spec.ts @@ -0,0 +1,51 @@ +import { GithubHttp } from '../../../util/http/github'; +import { CacheableGithubTags, FetchedTag } from './cache'; + +describe('modules/datasource/github-tags/cache', () => { + const http = new GithubHttp(); + const cache = new CacheableGithubTags(http, { resetDeltaMinutes: 0 }); + + const fetchedItem: FetchedTag = { + version: '1.2.3', + target: { + type: 'Commit', + hash: 'abc', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + }, + }; + + describe('coerceFetched', () => { + it('transforms GraphQL items', () => { + expect(cache.coerceFetched(fetchedItem)).toEqual({ + version: '1.2.3', + hash: 'abc', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + }); + expect( + cache.coerceFetched({ + version: '1.2.3', + target: { + type: 'Tag', + target: { + hash: 'abc', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + }, + }, + }) + ).toEqual({ + version: '1.2.3', + hash: 'abc', + releaseTimestamp: '2020-04-09T10:00:00.000Z', + }); + }); + + it('returns null for tags we can not process', () => { + expect( + cache.coerceFetched({ + version: '1.2.3', + target: { type: 'Blob' } as never, + }) + ).toBeNull(); + }); + }); +}); diff --git a/lib/modules/datasource/github-tags/cache.ts b/lib/modules/datasource/github-tags/cache.ts new file mode 100644 index 00000000000000..1f7d1fb407daf3 --- /dev/null +++ b/lib/modules/datasource/github-tags/cache.ts @@ -0,0 +1,88 @@ +import type { GithubHttp } from '../../../util/http/github'; +import { AbstractGithubDatasourceCache } from '../github-releases/cache/cache-base'; +import type { + CacheOptions, + StoredItemBase, +} from '../github-releases/cache/types'; + +const query = ` +query ($owner: String!, $name: String!, $cursor: String, $count: Int!) { + repository(owner: $owner, name: $name) { + payload: refs( + first: $count + after: $cursor + orderBy: {field: TAG_COMMIT_DATE, direction: DESC} + refPrefix: "refs/tags/" + ) { + nodes { + version: name + target { + type: __typename + ... on Commit { + hash: oid + releaseTimestamp: committedDate + } + ... on Tag { + target { + ... on Commit { + hash: oid + releaseTimestamp: committedDate + } + } + } + } + } + pageInfo { + hasNextPage + endCursor + } + } + } +} +`; + +export interface FetchedTag { + version: string; + target: + | { + type: 'Commit'; + hash: string; + releaseTimestamp: string; + } + | { + type: 'Tag'; + target: { + hash: string; + releaseTimestamp: string; + }; + }; +} + +export interface StoredTag extends StoredItemBase { + hash: string; + releaseTimestamp: string; +} + +export class CacheableGithubTags extends AbstractGithubDatasourceCache< + StoredTag, + FetchedTag +> { + readonly cacheNs = 'github-datasource-graphql-tags'; + readonly graphqlQuery = query; + + constructor(http: GithubHttp, opts: CacheOptions = {}) { + super(http, opts); + } + + coerceFetched(item: FetchedTag): StoredTag | null { + const { version, target } = item; + if (target.type === 'Commit') { + const { hash, releaseTimestamp } = target; + return { version, hash, releaseTimestamp }; + } else if (target.type === 'Tag') { + const { hash, releaseTimestamp } = target.target; + return { version, hash, releaseTimestamp }; + } + return null; + } +} diff --git a/lib/modules/datasource/github-tags/index.spec.ts b/lib/modules/datasource/github-tags/index.spec.ts index e8b978e5ae923f..e3315fa0844549 100644 --- a/lib/modules/datasource/github-tags/index.spec.ts +++ b/lib/modules/datasource/github-tags/index.spec.ts @@ -1,6 +1,8 @@ import { getPkgReleases } from '..'; import * as httpMock from '../../../../test/http-mock'; import * as _hostRules from '../../../util/host-rules'; +import { CacheableGithubReleases } from '../github-releases/cache'; +import { CacheableGithubTags } from './cache'; import { GithubTagsDatasource } from '.'; jest.mock('../../../util/host-rules'); @@ -10,6 +12,15 @@ const githubApiHost = 'https://api.github.com'; const githubEnterpriseApiHost = 'https://git.enterprise.com'; describe('modules/datasource/github-tags/index', () => { + const releasesCacheGetItems = jest.spyOn( + CacheableGithubReleases.prototype, + 'getItems' + ); + const tagsCacheGetItems = jest.spyOn( + CacheableGithubTags.prototype, + 'getItems' + ); + const github = new GithubTagsDatasource(); beforeEach(() => { @@ -116,37 +127,38 @@ describe('modules/datasource/github-tags/index', () => { const depName = 'some/dep2'; it('returns tags', async () => { - const tags = [{ name: 'v1.0.0' }, { name: 'v1.1.0' }]; - const releases = tags.map((item, idx) => ({ - tag_name: item.name, - published_at: new Date(idx), - prerelease: !!idx, - })); - httpMock - .scope(githubApiHost) - .get(`/repos/${depName}/tags?per_page=100`) - .reply(200, tags) - .get(`/repos/${depName}/releases?per_page=100`) - .reply(200, releases); + tagsCacheGetItems.mockResolvedValueOnce([ + { version: 'v1.0.0', releaseTimestamp: '2021-01-01', hash: '123' }, + { version: 'v2.0.0', releaseTimestamp: '2022-01-01', hash: 'abc' }, + ]); + releasesCacheGetItems.mockResolvedValueOnce([ + { version: 'v1.0.0', releaseTimestamp: '2021-01-01', isStable: true }, + { version: 'v2.0.0', releaseTimestamp: '2022-01-01', isStable: false }, + ] as never); + const res = await getPkgReleases({ datasource: github.id, depName }); - expect(res).toMatchSnapshot(); - expect(res.releases).toHaveLength(2); - }); - it('supports ghe', async () => { - const body = [{ name: 'v1.0.0' }, { name: 'v1.1.0' }]; - httpMock - .scope(githubEnterpriseApiHost) - .get(`/api/v3/repos/${depName}/tags?per_page=100`) - .reply(200, body) - .get(`/api/v3/repos/${depName}/releases?per_page=100`) - .reply(404); - - const res = await github.getReleases({ - registryUrl: 'https://git.enterprise.com', - packageName: depName, + expect(res).toEqual({ + registryUrl: 'https://github.com', + sourceUrl: 'https://github.com/some/dep2', + releases: [ + { + gitRef: 'v1.0.0', + hash: '123', + isStable: true, + releaseTimestamp: '2021-01-01T00:00:00.000Z', + version: 'v1.0.0', + }, + + { + gitRef: 'v2.0.0', + hash: 'abc', + isStable: false, + releaseTimestamp: '2022-01-01T00:00:00.000Z', + version: 'v2.0.0', + }, + ], }); - expect(res).toMatchSnapshot(); }); }); }); diff --git a/lib/modules/datasource/github-tags/index.ts b/lib/modules/datasource/github-tags/index.ts index e54974d2d5c518..03a3bb573c35ab 100644 --- a/lib/modules/datasource/github-tags/index.ts +++ b/lib/modules/datasource/github-tags/index.ts @@ -8,13 +8,17 @@ import type { Release, ReleaseResult, } from '../types'; -import type { GitHubTag, TagResponse } from './types'; +import { CacheableGithubTags } from './cache'; +import type { TagResponse } from './types'; export class GithubTagsDatasource extends GithubReleasesDatasource { static override readonly id = 'github-tags'; + private tagsCache: CacheableGithubTags; + constructor() { super(GithubTagsDatasource.id); + this.tagsCache = new CacheableGithubTags(this.http); } @cache({ @@ -93,45 +97,21 @@ export class GithubTagsDatasource extends GithubReleasesDatasource { this.getCommit(registryUrl, repo!); } - @cache({ - ttlMinutes: 10, - namespace: `datasource-${GithubTagsDatasource.id}`, - key: ({ registryUrl, packageName: repo }: GetReleasesConfig) => - `${registryUrl}:${repo}:tags`, - }) - async getTags({ - registryUrl, - packageName: repo, - }: GetReleasesConfig): Promise { - const apiBaseUrl = getApiBaseUrl(registryUrl); - // tag - const url = `${apiBaseUrl}repos/${repo}/tags?per_page=100`; - - const versions = ( - await this.http.getJson(url, { - paginate: true, - }) - ).body.map((o) => o.name); - const dependency: ReleaseResult = { - sourceUrl: getSourceUrl(repo, registryUrl), - releases: versions.map((version) => ({ - version, - gitRef: version, - })), - }; - return dependency; - } - override async getReleases( config: GetReleasesConfig ): Promise { - const tagsResult = await this.getTags(config); + const tagReleases = await this.tagsCache.getItems(config); // istanbul ignore if - if (!tagsResult) { + if (!tagReleases.length) { return null; } + const tagsResult: ReleaseResult = { + sourceUrl: getSourceUrl(config.packageName, config.registryUrl), + releases: tagReleases.map((item) => ({ ...item, gitRef: item.version })), + }; + try { // Fetch additional data from releases endpoint when possible const releasesResult = await super.getReleases(config); diff --git a/lib/modules/datasource/go/__snapshots__/releases-direct.spec.ts.snap b/lib/modules/datasource/go/__snapshots__/releases-direct.spec.ts.snap index 80e885cb4ee953..5d82c767a1eaf0 100644 --- a/lib/modules/datasource/go/__snapshots__/releases-direct.spec.ts.snap +++ b/lib/modules/datasource/go/__snapshots__/releases-direct.spec.ts.snap @@ -1,21 +1,5 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`modules/datasource/go/releases-direct getReleases processes real data 1`] = ` -Object { - "releases": Array [ - Object { - "gitRef": "v1.0.0", - "version": "v1.0.0", - }, - Object { - "gitRef": "v2.0.0", - "version": "v2.0.0", - }, - ], - "sourceUrl": "https://github.com/golang/text", -} -`; - exports[`modules/datasource/go/releases-direct getReleases support bitbucket tags 1`] = ` Object { "registryUrl": "https://bitbucket.org", @@ -35,22 +19,6 @@ Object { } `; -exports[`modules/datasource/go/releases-direct getReleases support ghe 1`] = ` -Object { - "releases": Array [ - Object { - "gitRef": "v1.0.0", - "version": "v1.0.0", - }, - Object { - "gitRef": "v2.0.0", - "version": "v2.0.0", - }, - ], - "sourceUrl": "https://git.enterprise.com/example/module", -} -`; - exports[`modules/datasource/go/releases-direct getReleases support gitlab 1`] = ` Object { "releases": Array [ diff --git a/lib/modules/datasource/go/releases-direct.spec.ts b/lib/modules/datasource/go/releases-direct.spec.ts index 6c6c1159e2f41e..5753655146d089 100644 --- a/lib/modules/datasource/go/releases-direct.spec.ts +++ b/lib/modules/datasource/go/releases-direct.spec.ts @@ -1,6 +1,7 @@ import * as httpMock from '../../../../test/http-mock'; import { mocked } from '../../../../test/util'; import * as _hostRules from '../../../util/host-rules'; +import { GithubTagsDatasource } from '../github-tags'; import { BaseGoDatasource } from './base'; import { GoDirectDatasource } from './releases-direct'; @@ -12,6 +13,11 @@ const getDatasourceSpy = jest.spyOn(BaseGoDatasource, 'getDatasource'); const hostRules = mocked(_hostRules); describe('modules/datasource/go/releases-direct', () => { + const githubGetTags = jest.spyOn( + GithubTagsDatasource.prototype, + 'getReleases' + ); + beforeEach(() => { jest.resetAllMocks(); hostRules.find.mockReturnValue({}); @@ -46,18 +52,24 @@ describe('modules/datasource/go/releases-direct', () => { packageName: 'golang/text', registryUrl: 'https://github.com', }); - httpMock - .scope('https://api.github.com/') - .get('/repos/golang/text/tags?per_page=100') - .reply(200, [{ name: 'v1.0.0' }, { name: 'v2.0.0' }]) - .get('/repos/golang/text/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValueOnce({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v2.0.0', version: 'v2.0.0' }, + ], + }); + const res = await datasource.getReleases({ packageName: 'golang.org/x/text', }); - expect(res).toMatchSnapshot(); - expect(res).not.toBeNull(); - expect(res).toBeDefined(); + + expect(res).toEqual({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v2.0.0', version: 'v2.0.0' }, + ], + sourceUrl: 'https://github.com/golang/text', + }); }); it('support gitlab', async () => { @@ -125,18 +137,27 @@ describe('modules/datasource/go/releases-direct', () => { registryUrl: 'https://git.enterprise.com', packageName: 'example/module', }); - httpMock - .scope('https://git.enterprise.com/') - .get('/api/v3/repos/example/module/tags?per_page=100') - .reply(200, [{ name: 'v1.0.0' }, { name: 'v2.0.0' }]) - .get('/api/v3/repos/example/module/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValueOnce({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v2.0.0', version: 'v2.0.0' }, + ], + }); + const res = await datasource.getReleases({ packageName: 'git.enterprise.com/example/module', }); - expect(res).toMatchSnapshot(); - expect(res).not.toBeNull(); - expect(res).toBeDefined(); + + expect(res).toEqual({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v2.0.0', version: 'v2.0.0' }, + ], + sourceUrl: 'https://git.enterprise.com/example/module', + }); + expect(githubGetTags.mock.calls).toMatchObject([ + [{ registryUrl: 'https://git.enterprise.com' }], + ]); }); it('works for known servers', async () => { @@ -155,20 +176,7 @@ describe('modules/datasource/go/releases-direct', () => { packageName: 'go-x/x', registryUrl: 'https://github.com', }); - httpMock - .scope('https://api.github.com/') - .get('/repos/x/text/tags?per_page=100') - .reply(200, []) - .get('/repos/x/text/releases?per_page=100') - .reply(200, []) - .get('/repos/x/text/tags?per_page=100') - .reply(200, []) - .get('/repos/x/text/releases?per_page=100') - .reply(200, []) - .get('/repos/go-x/x/tags?per_page=100') - .reply(200, []) - .get('/repos/go-x/x/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValue({ releases: [] }); const packages = [ { packageName: 'github.com/x/text' }, { packageName: 'gopkg.in/x/text' }, @@ -178,6 +186,7 @@ describe('modules/datasource/go/releases-direct', () => { const res = await datasource.getReleases(pkg); expect(res.releases).toBeEmpty(); } + expect(githubGetTags).toHaveBeenCalledTimes(3); }); it('support gitlab subgroups', async () => { @@ -220,16 +229,15 @@ describe('modules/datasource/go/releases-direct', () => { { packageName: 'github.com/x/text/a' }, { packageName: 'github.com/x/text/b' }, ]; - const tags = [{ name: 'a/v1.0.0' }, { name: 'b/v2.0.0' }]; - for (const pkg of packages) { - httpMock - .scope('https://api.github.com/') - .get('/repos/x/text/tags?per_page=100') - .reply(200, tags) - .get('/repos/x/text/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValue({ + releases: [ + { version: 'a/v1.0.0', gitRef: 'a/v1.0.0' }, + { version: 'b/v2.0.0', gitRef: 'b/v2.0.0' }, + ], + }); + for (const pkg of packages) { const prefix = pkg.packageName.split('/')[3]; const result = await datasource.getReleases(pkg); expect(result.releases).toHaveLength(1); @@ -252,16 +260,15 @@ describe('modules/datasource/go/releases-direct', () => { { packageName: 'github.com/x/text/a' }, { packageName: 'github.com/x/text/b' }, ]; - const tags = [{ name: 'v1.0.0' }, { name: 'v2.0.0' }]; - for (const pkg of packages) { - httpMock - .scope('https://api.github.com/') - .get('/repos/x/text/tags?per_page=100') - .reply(200, tags) - .get('/repos/x/text/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValue({ + releases: [ + { version: 'v1.0.0', gitRef: 'v1.0.0' }, + { version: 'v2.0.0', gitRef: 'v2.0.0' }, + ], + }); + for (const pkg of packages) { const result = await datasource.getReleases(pkg); expect(result.releases).toHaveLength(0); } @@ -274,24 +281,20 @@ describe('modules/datasource/go/releases-direct', () => { registryUrl: 'https://github.com', }); const pkg = { packageName: 'github.com/x/text/b/v2' }; - const tags = [ - { name: 'a/v1.0.0' }, - { name: 'v5.0.0' }, - { name: 'b/v2.0.0' }, - { name: 'b/v3.0.0' }, - ]; - httpMock - .scope('https://api.github.com/') - .get('/repos/x/text/tags?per_page=100') - .reply(200, tags) - .get('/repos/x/text/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValue({ + releases: [ + { version: 'a/v1.0.0', gitRef: 'a/v1.0.0' }, + { version: 'v5.0.0', gitRef: 'v5.0.0' }, + { version: 'b/v2.0.0', gitRef: 'b/v2.0.0' }, + { version: 'b/v3.0.0', gitRef: 'b/v3.0.0' }, + ], + }); const result = await datasource.getReleases(pkg); expect(result.releases).toEqual([ - { gitRef: 'b/v2.0.0', version: 'v2.0.0' }, - { gitRef: 'b/v3.0.0', version: 'v3.0.0' }, + { version: 'v2.0.0', gitRef: 'b/v2.0.0' }, + { version: 'v3.0.0', gitRef: 'b/v3.0.0' }, ]); }); }); diff --git a/lib/modules/datasource/go/releases-goproxy.spec.ts b/lib/modules/datasource/go/releases-goproxy.spec.ts index a2bbec4a9d5de4..9936e8d1bf7bd2 100644 --- a/lib/modules/datasource/go/releases-goproxy.spec.ts +++ b/lib/modules/datasource/go/releases-goproxy.spec.ts @@ -1,10 +1,22 @@ import * as httpMock from '../../../../test/http-mock'; import { loadFixture } from '../../../../test/util'; +import { GithubReleasesDatasource } from '../github-releases'; +import { GithubTagsDatasource } from '../github-tags'; import { GoProxyDatasource } from './releases-goproxy'; const datasource = new GoProxyDatasource(); describe('modules/datasource/go/releases-goproxy', () => { + const githubGetReleases = jest.spyOn( + GithubReleasesDatasource.prototype, + 'getReleases' + ); + + const githubGetTags = jest.spyOn( + GithubTagsDatasource.prototype, + 'getReleases' + ); + it('encodeCase', () => { expect(datasource.encodeCase('foo')).toBe('foo'); expect(datasource.encodeCase('Foo')).toBe('!foo'); @@ -276,12 +288,13 @@ describe('modules/datasource/go/releases-goproxy', () => { process.env.GOPROXY = baseUrl; process.env.GOPRIVATE = 'github.com/google/*'; - httpMock - .scope('https://api.github.com/') - .get('/repos/google/btree/tags?per_page=100') - .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }]) - .get('/repos/google/btree/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValueOnce({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v1.0.1', version: 'v1.0.1' }, + ], + }); + githubGetReleases.mockResolvedValueOnce({ releases: [] }); const res = await datasource.getReleases({ packageName: 'github.com/google/btree', @@ -458,12 +471,13 @@ describe('modules/datasource/go/releases-goproxy', () => { .get('/@v/list') .reply(410); - httpMock - .scope('https://api.github.com/') - .get('/repos/foo/bar/tags?per_page=100') - .reply(200, [{ name: 'v1.0.0' }, { name: 'v1.0.1' }]) - .get('/repos/foo/bar/releases?per_page=100') - .reply(200, []); + githubGetTags.mockResolvedValueOnce({ + releases: [ + { gitRef: 'v1.0.0', version: 'v1.0.0' }, + { gitRef: 'v1.0.1', version: 'v1.0.1' }, + ], + }); + githubGetReleases.mockResolvedValueOnce({ releases: [] }); const res = await datasource.getReleases({ packageName: 'github.com/foo/bar', diff --git a/lib/util/http/github.ts b/lib/util/http/github.ts index aa39594e043293..e4b43912763106 100644 --- a/lib/util/http/github.ts +++ b/lib/util/http/github.ts @@ -45,7 +45,7 @@ interface GithubGraphqlRepoData { repository?: T; } -interface GithubGraphqlResponse { +export interface GithubGraphqlResponse { data?: T; errors?: { type?: string; diff --git a/lib/workers/repository/process/lookup/index.spec.ts b/lib/workers/repository/process/lookup/index.spec.ts index 3c2a480f320839..090143ceb9f06e 100644 --- a/lib/workers/repository/process/lookup/index.spec.ts +++ b/lib/workers/repository/process/lookup/index.spec.ts @@ -19,22 +19,6 @@ import * as lookup from '.'; jest.mock('../../../../modules/datasource/docker'); -jest.mock('../../../../modules/datasource/git-refs', function () { - const { GitRefsDatasource: Orig } = jest.requireActual( - '../../../../modules/datasource/git-refs' - ); - const Mocked = jest.fn().mockImplementation(() => ({ - getReleases: () => - Promise.resolve({ - releases: [{ version: 'master' }], - }), - getDigest: () => - Promise.resolve('4b825dc642cb6eb9a060e54bf8d69288fbee4904'), - })); - Mocked['id'] = Orig.id; - return { GitRefsDatasource: Mocked }; -}); - const fixtureRoot = '../../../../config/npm'; const qJson = { ...Fixtures.getJson('01.json', fixtureRoot), @@ -53,6 +37,11 @@ const docker = mocked(DockerDatasource.prototype); let config: LookupUpdateConfig; describe('workers/repository/process/lookup/index', () => { + const getGithubReleases = jest.spyOn( + GithubReleasesDatasource.prototype, + 'getReleases' + ); + beforeEach(() => { // TODO: fix types #7154 config = partial(getConfig() as never); @@ -60,6 +49,14 @@ describe('workers/repository/process/lookup/index', () => { config.versioning = npmVersioningId; config.rangeStrategy = 'replace'; jest.resetAllMocks(); + jest + .spyOn(GitRefsDatasource.prototype, 'getReleases') + .mockResolvedValueOnce({ + releases: [{ version: 'master' }], + }); + jest + .spyOn(GitRefsDatasource.prototype, 'getDigest') + .mockResolvedValueOnce('4b825dc642cb6eb9a060e54bf8d69288fbee4904'); }); // TODO: fix mocks @@ -884,14 +881,13 @@ describe('workers/repository/process/lookup/index', () => { config.currentValue = '1.4.4'; config.depName = 'some/action'; config.datasource = GithubReleasesDatasource.id; - httpMock - .scope('https://api.github.com') - .get('/repos/some/action/releases?per_page=100') - .reply(200, [ - { tag_name: '1.4.4' }, - { tag_name: '2.0.0' }, - { tag_name: '2.1.0', prerelease: true }, - ]); + getGithubReleases.mockResolvedValueOnce({ + releases: [ + { version: '1.4.4' }, + { version: '2.0.0' }, + { version: '2.1.0', isStable: false }, + ], + }); expect((await lookup.lookupUpdates(config)).updates).toMatchSnapshot([ { newValue: '2.0.0', updateType: 'major' }, ]); @@ -907,14 +903,13 @@ describe('workers/repository/process/lookup/index', () => { yesterday.setDate(yesterday.getDate() - 1); const lastWeek = new Date(); lastWeek.setDate(lastWeek.getDate() - 7); - httpMock - .scope('https://api.github.com') - .get('/repos/some/action/releases?per_page=100') - .reply(200, [ - { tag_name: '1.4.4' }, - { tag_name: '1.4.5', published_at: lastWeek.toISOString() }, - { tag_name: '1.4.6', published_at: yesterday.toISOString() }, - ]); + getGithubReleases.mockResolvedValueOnce({ + releases: [ + { version: '1.4.4' }, + { version: '1.4.5', releaseTimestamp: lastWeek.toISOString() }, + { version: '1.4.6', releaseTimestamp: yesterday.toISOString() }, + ], + }); const res = await lookup.lookupUpdates(config); expect(res.updates).toHaveLength(1); expect(res.updates[0].newVersion).toBe('1.4.6'); @@ -931,14 +926,13 @@ describe('workers/repository/process/lookup/index', () => { yesterday.setDate(yesterday.getDate() - 1); const lastWeek = new Date(); lastWeek.setDate(lastWeek.getDate() - 7); - httpMock - .scope('https://api.github.com') - .get('/repos/some/action/releases?per_page=100') - .reply(200, [ - { tag_name: '1.4.4' }, - { tag_name: '1.4.5', published_at: lastWeek.toISOString() }, - { tag_name: '1.4.6', published_at: yesterday.toISOString() }, - ]); + getGithubReleases.mockResolvedValueOnce({ + releases: [ + { version: '1.4.4' }, + { version: '1.4.5', releaseTimestamp: lastWeek.toISOString() }, + { version: '1.4.6', releaseTimestamp: yesterday.toISOString() }, + ], + }); const res = await lookup.lookupUpdates(config); expect(res.updates).toHaveLength(1); expect(res.updates[0].newVersion).toBe('1.4.5'); diff --git a/lib/workers/repository/update/pr/changelog/github.spec.ts b/lib/workers/repository/update/pr/changelog/github.spec.ts index 0b36cb106f70ea..313d4d55d2bda0 100644 --- a/lib/workers/repository/update/pr/changelog/github.spec.ts +++ b/lib/workers/repository/update/pr/changelog/github.spec.ts @@ -1,6 +1,7 @@ import * as httpMock from '../../../../../../test/http-mock'; import { GlobalConfig } from '../../../../../config/global'; import { PlatformId } from '../../../../../constants'; +import { CacheableGithubTags } from '../../../../../modules/datasource/github-tags/cache'; import * as semverVersioning from '../../../../../modules/versioning/semver'; import * as hostRules from '../../../../../util/host-rules'; import type { BranchUpgradeConfig } from '../../../../types'; @@ -35,6 +36,7 @@ describe('workers/repository/update/pr/changelog/github', () => { afterEach(() => { // FIXME: add missing http mocks httpMock.clear(false); + jest.resetAllMocks(); }); describe('getChangeLogJSON', () => { @@ -297,15 +299,17 @@ describe('workers/repository/update/pr/changelog/github', () => { }); it('works with same version releases but different prefix', async () => { - httpMock - .scope('https://api.github.com/') - .get('/repos/chalk/chalk/tags?per_page=100') - .reply(200, [ - { name: 'v1.0.1' }, - { name: '1.0.1' }, - { name: 'correctPrefix/target@1.0.1' }, - { name: 'wrongPrefix/target-1.0.1' }, - ]); + const githubTagsMock = jest.spyOn( + CacheableGithubTags.prototype, + 'getItems' + ); + + githubTagsMock.mockResolvedValue([ + { version: 'v1.0.1' }, + { version: '1.0.1' }, + { version: 'correctPrefix/target@1.0.1' }, + { version: 'wrongPrefix/target-1.0.1' }, + ] as never); const upgradeData: BranchUpgradeConfig = { manager: 'some-manager', diff --git a/lib/workers/repository/update/pr/changelog/github/index.ts b/lib/workers/repository/update/pr/changelog/github/index.ts index 4ced3f393b1c19..5f8817ef7d2a9a 100644 --- a/lib/workers/repository/update/pr/changelog/github/index.ts +++ b/lib/workers/repository/update/pr/changelog/github/index.ts @@ -1,7 +1,7 @@ import changelogFilenameRegex from 'changelog-filename-regex'; import { logger } from '../../../../../../logger'; -import type { GithubRelease } from '../../../../../../modules/datasource/github-releases/types'; -import type { GitHubTag } from '../../../../../../modules/datasource/github-tags/types'; +import { CacheableGithubReleases } from '../../../../../../modules/datasource/github-releases/cache'; +import { CacheableGithubTags } from '../../../../../../modules/datasource/github-tags/cache'; import type { GithubGitBlob, GithubGitTree, @@ -14,25 +14,26 @@ import type { ChangeLogFile, ChangeLogNotes } from '../types'; export const id = 'github-changelog'; const http = new GithubHttp(id); +const tagsCache = new CacheableGithubTags(http); +const releasesCache = new CacheableGithubReleases(http); export async function getTags( endpoint: string, repository: string ): Promise { logger.trace('github.getTags()'); - const url = `${endpoint}repos/${repository}/tags?per_page=100`; try { - const res = await http.getJson(url, { - paginate: true, + const tags = await tagsCache.getItems({ + registryUrl: endpoint, + packageName: repository, }); - const tags = res.body; - + // istanbul ignore if if (!tags.length) { logger.debug({ repository }, 'repository has no Github tags'); } - return tags.map((tag) => tag.name).filter(Boolean); + return tags.map(({ version }) => version).filter(Boolean); } catch (err) { logger.debug( { sourceRepo: repository, err }, @@ -110,16 +111,19 @@ export async function getReleaseList( repository: string ): Promise { logger.trace('github.getReleaseList()'); - const url = `${ensureTrailingSlash(apiBaseUrl)}repos/${repository}/releases`; - const res = await http.getJson(`${url}?per_page=100`, { - paginate: true, + const notesSourceUrl = `${ensureTrailingSlash( + apiBaseUrl + )}repos/${repository}/releases`; + const items = await releasesCache.getItems({ + registryUrl: apiBaseUrl, + packageName: repository, }); - return res.body.map((release) => ({ - url: release.html_url, - notesSourceUrl: url, - id: release.id, - tag: release.tag_name, - name: release.name, - body: release.body, + return items.map(({ url, id, version: tag, name, description: body }) => ({ + url, + notesSourceUrl, + id, + tag, + name, + body, })); } diff --git a/lib/workers/repository/update/pr/changelog/index.spec.ts b/lib/workers/repository/update/pr/changelog/index.spec.ts index 67139d656665bc..570c613e9cebbf 100644 --- a/lib/workers/repository/update/pr/changelog/index.spec.ts +++ b/lib/workers/repository/update/pr/changelog/index.spec.ts @@ -2,6 +2,8 @@ import * as httpMock from '../../../../../../test/http-mock'; import { partial } from '../../../../../../test/util'; import { GlobalConfig } from '../../../../../config/global'; import { PlatformId } from '../../../../../constants'; +import { CacheableGithubReleases } from '../../../../../modules/datasource/github-releases/cache'; +import { CacheableGithubTags } from '../../../../../modules/datasource/github-tags/cache'; import * as semverVersioning from '../../../../../modules/versioning/semver'; import * as hostRules from '../../../../../util/host-rules'; import type { BranchConfig } from '../../../../types'; @@ -34,7 +36,17 @@ const upgrade: BranchConfig = partial({ describe('workers/repository/update/pr/changelog/index', () => { describe('getChangeLogJSON', () => { + const githubReleasesMock = jest.spyOn( + CacheableGithubReleases.prototype, + 'getItems' + ); + const githubTagsMock = jest.spyOn( + CacheableGithubTags.prototype, + 'getItems' + ); + beforeEach(() => { + jest.resetAllMocks(); hostRules.clear(); hostRules.add({ hostType: PlatformId.Github, @@ -81,15 +93,12 @@ describe('workers/repository/update/pr/changelog/index', () => { }); it('works without Github', async () => { + githubTagsMock.mockRejectedValueOnce(new Error('Unknown')); + githubReleasesMock.mockRejectedValueOnce(new Error('Unknown')); httpMock .scope(githubApiHost) .get('/repos/chalk/chalk') .times(4) - .reply(500) - .get('/repos/chalk/chalk/tags?per_page=100') - .reply(500) - .get('/repos/chalk/chalk/releases?per_page=100') - .times(4) .reply(500); expect( await getChangeLogJSON({ @@ -116,20 +125,16 @@ describe('workers/repository/update/pr/changelog/index', () => { }); it('uses GitHub tags', async () => { - httpMock - .scope(githubApiHost) - .get('/repos/chalk/chalk/tags?per_page=100') - .reply(200, [ - { name: '0.9.0' }, - { name: '1.0.0' }, - { name: '1.4.0' }, - { name: 'v2.3.0' }, - { name: '2.2.2' }, - { name: 'v2.4.2' }, - ]) - .persist() - .get(/.*/) - .reply(200, []); + httpMock.scope(githubApiHost).get(/.*/).reply(200, []).persist(); + githubTagsMock.mockResolvedValue([ + { version: '0.9.0' }, + { version: '1.0.0' }, + { version: '1.4.0' }, + { version: 'v2.3.0' }, + { version: '2.2.2' }, + { version: 'v2.4.2' }, + ] as never); + githubReleasesMock.mockResolvedValue([]); expect( await getChangeLogJSON({ ...upgrade, @@ -155,11 +160,11 @@ describe('workers/repository/update/pr/changelog/index', () => { }); it('filters unnecessary warns', async () => { - httpMock - .scope(githubApiHost) - .persist() - .get(/.*/) - .replyWithError('Unknown Github Repo'); + githubTagsMock.mockRejectedValueOnce(new Error('Unknown Github Repo')); + githubReleasesMock.mockRejectedValueOnce( + new Error('Unknown Github Repo') + ); + httpMock.scope(githubApiHost).get(/.*/).reply(200, []).persist(); const res = await getChangeLogJSON({ ...upgrade, depName: '@renovate/no', @@ -185,6 +190,8 @@ describe('workers/repository/update/pr/changelog/index', () => { }); it('supports node engines', async () => { + githubTagsMock.mockRejectedValueOnce([]); + githubReleasesMock.mockRejectedValueOnce([]); expect( await getChangeLogJSON({ ...upgrade, @@ -259,6 +266,8 @@ describe('workers/repository/update/pr/changelog/index', () => { }); it('supports github enterprise and github.com changelog', async () => { + githubTagsMock.mockRejectedValueOnce([]); + githubReleasesMock.mockRejectedValueOnce([]); httpMock.scope(githubApiHost).persist().get(/.*/).reply(200, []); hostRules.add({ hostType: PlatformId.Github, @@ -291,6 +300,8 @@ describe('workers/repository/update/pr/changelog/index', () => { }); it('supports github enterprise and github enterprise changelog', async () => { + githubTagsMock.mockRejectedValueOnce([]); + githubReleasesMock.mockRejectedValueOnce([]); httpMock .scope('https://github-enterprise.example.com') .persist() @@ -329,6 +340,8 @@ describe('workers/repository/update/pr/changelog/index', () => { }); it('supports github.com and github enterprise changelog', async () => { + githubTagsMock.mockRejectedValueOnce([]); + githubReleasesMock.mockRejectedValueOnce([]); httpMock .scope('https://github-enterprise.example.com') .persist() diff --git a/lib/workers/repository/update/pr/changelog/release-notes.spec.ts b/lib/workers/repository/update/pr/changelog/release-notes.spec.ts index 04c99e669e325a..aa94a95e66515e 100644 --- a/lib/workers/repository/update/pr/changelog/release-notes.spec.ts +++ b/lib/workers/repository/update/pr/changelog/release-notes.spec.ts @@ -1,6 +1,7 @@ import { DateTime } from 'luxon'; import * as httpMock from '../../../../../../test/http-mock'; import { loadFixture, mocked } from '../../../../../../test/util'; +import { CacheableGithubReleases } from '../../../../../modules/datasource/github-releases/cache'; import { clone } from '../../../../../util/clone'; import * as _hostRules from '../../../../../util/host-rules'; import { toBase64 } from '../../../../../util/string'; @@ -56,6 +57,11 @@ const gitlabProject = { } as ChangeLogProject; describe('workers/repository/update/pr/changelog/release-notes', () => { + const githubReleasesMock = jest.spyOn( + CacheableGithubReleases.prototype, + 'getItems' + ); + beforeEach(() => { hostRules.find.mockReturnValue({}); hostRules.hosts.mockReturnValue([]); @@ -161,16 +167,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { }); it('should return release list for github repo', async () => { - httpMock - .scope('https://api.github.com/') - .get('/repos/some/yet-other-repository/releases?per_page=100') - .reply(200, [ - { tag_name: `v1.0.0` }, - { - tag_name: `v1.0.1`, - body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', - }, - ]); + githubReleasesMock.mockResolvedValueOnce([ + { version: `v1.0.0` }, + { + version: `v1.0.1`, + description: + 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', + }, + ] as never); const res = await getReleaseList({ ...githubProject, @@ -263,10 +267,10 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { describe('getReleaseNotes()', () => { it('should return null for release notes without body', async () => { - httpMock - .scope('https://api.github.com/') - .get('/repos/some/repository/releases?per_page=100') - .reply(200, [{ tag_name: 'v1.0.0' }, { tag_name: 'v1.0.1' }]); + githubReleasesMock.mockResolvedValueOnce([ + { version: 'v1.0.0' }, + { version: 'v1.0.1' }, + ] as never); const res = await getReleaseNotes( { ...githubProject, @@ -279,17 +283,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { }); it('gets release notes with body ""', async () => { - const prefix = ''; - httpMock - .scope('https://api.github.com/') - .get('/repos/some/other-repository/releases?per_page=100') - .reply(200, [ - { tag_name: `${prefix}1.0.0` }, - { - tag_name: `${prefix}1.0.1`, - body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', - }, - ]); + githubReleasesMock.mockResolvedValueOnce([ + { version: '1.0.0' }, + { + version: '1.0.1', + description: + 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', + }, + ] as never); const res = await getReleaseNotes( { ...githubProject, @@ -310,17 +311,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { }); it('gets release notes with body "v"', async () => { - const prefix = 'v'; - httpMock - .scope('https://api.github.com/') - .get('/repos/some/other-repository/releases?per_page=100') - .reply(200, [ - { tag_name: `${prefix}1.0.0` }, - { - tag_name: `${prefix}1.0.1`, - body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', - }, - ]); + githubReleasesMock.mockResolvedValueOnce([ + { version: 'v1.0.0' }, + { + version: 'v1.0.1', + description: + 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', + }, + ] as never); const res = await getReleaseNotes( { ...githubProject, @@ -341,17 +339,15 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { }); it('gets release notes with body "other-"', async () => { - const prefix = 'other-'; - httpMock - .scope('https://api.github.com/') - .get('/repos/some/other-repository/releases?per_page=100') - .reply(200, [ - { tag_name: `${prefix}1.0.0` }, - { - tag_name: `${prefix}1.0.1`, - body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', - }, - ]); + githubReleasesMock.mockResolvedValueOnce([ + { version: 'other-1.0.0' }, + { + version: 'other-1.0.1', + description: + 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', + }, + ] as never); + const res = await getReleaseNotes( { ...githubProject, @@ -372,17 +368,15 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { }); it('gets release notes with body "other_v"', async () => { - const prefix = 'other_v'; - httpMock - .scope('https://api.github.com/') - .get('/repos/some/other-repository/releases?per_page=100') - .reply(200, [ - { tag_name: `${prefix}1.0.0` }, - { - tag_name: `${prefix}1.0.1`, - body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', - }, - ]); + githubReleasesMock.mockResolvedValueOnce([ + { version: 'other_v1.0.0' }, + { + version: 'other_v1.0.1', + description: + 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', + }, + ] as never); + const res = await getReleaseNotes( { ...githubProject, @@ -403,17 +397,14 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { }); it('gets release notes with body "other@"', async () => { - const prefix = 'other@'; - httpMock - .scope('https://api.github.com/') - .get('/repos/some/other-repository/releases?per_page=100') - .reply(200, [ - { tag_name: `${prefix}1.0.0` }, - { - tag_name: `${prefix}1.0.1`, - body: 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', - }, - ]); + githubReleasesMock.mockResolvedValueOnce([ + { version: 'other@1.0.0' }, + { + version: 'other@1.0.1', + description: + 'some body #123, [#124](https://github.com/some/yet-other-repository/issues/124)', + }, + ] as never); const res = await getReleaseNotes( { ...githubProject, @@ -547,20 +538,17 @@ describe('workers/repository/update/pr/changelog/release-notes', () => { it('handles same version but different repo releases', async () => { const depName = 'correctTagPrefix/exampleDep'; - httpMock - .scope('https://api.github.com/') - .get('/repos/some/other-repository/releases?per_page=100') - .reply(200, [ - { - tag_name: `${depName}@1.0.0`, - html_url: 'correct/url/tag.com', - body: 'some body', - }, - { tag_name: `someOtherRelease1/exampleDep_1.0.0` }, - { - tag_name: `someOtherRelease2/exampleDep-1.0.0`, - }, - ]); + githubReleasesMock.mockResolvedValueOnce([ + { + version: `${depName}@1.0.0`, + url: 'correct/url/tag.com', + description: 'some body', + }, + { version: `someOtherRelease1/exampleDep_1.0.0` }, + { + version: `someOtherRelease2/exampleDep-1.0.0`, + }, + ] as never); const res = await getReleaseNotes( { ...githubProject,