From e4e76f8f49770b8d14c6bc289038ba74ff27a6b9 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 23 Jun 2020 14:40:26 +0200 Subject: [PATCH] refactor: simplify crate datasource --- .../crate/__fixtures__/invalid_crate_data | 1 - .../crate/__snapshots__/index.spec.ts.snap | 181 ------------------ lib/datasource/crate/index.spec.ts | 22 --- lib/datasource/crate/index.ts | 64 ++----- 4 files changed, 19 insertions(+), 249 deletions(-) delete mode 100644 lib/datasource/crate/__fixtures__/invalid_crate_data diff --git a/lib/datasource/crate/__fixtures__/invalid_crate_data b/lib/datasource/crate/__fixtures__/invalid_crate_data deleted file mode 100644 index 6de95ab075a2ae..00000000000000 --- a/lib/datasource/crate/__fixtures__/invalid_crate_data +++ /dev/null @@ -1 +0,0 @@ -{"name":"some_crate","deps":[{"name":"time","req":"^0.1","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"},{"name":"threadpool","req":"^0.1.4","features":[],"optional":false,"default_features":true,"target":null,"kind":"normal"}],"cksum":"2884e8cf8d78d9a6de8bbc1898603edb2b625eb8f64e371919906d0fec5660e7","features":{},"yanked":false} \ No newline at end of file diff --git a/lib/datasource/crate/__snapshots__/index.spec.ts.snap b/lib/datasource/crate/__snapshots__/index.spec.ts.snap index f5d2a6e7250c73..b16d093225e8b9 100644 --- a/lib/datasource/crate/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/crate/__snapshots__/index.spec.ts.snap @@ -501,101 +501,6 @@ Array [ ] `; -exports[`datasource/crate getReleases returns null for invalid crate data 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/li/bc/libc", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/am/et/amethyst", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/in/va/invalid-crate-name", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, -] -`; - exports[`datasource/crate getReleases returns null for missing fields 1`] = ` Array [ Object { @@ -678,92 +583,6 @@ Array [ ] `; -exports[`datasource/crate getReleases returns null if crate name is invalid 1`] = ` -Array [ - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/no/n_/non_existent_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/so/me/some_crate", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/li/bc/libc", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/am/et/amethyst", - }, - Object { - "headers": Object { - "accept-encoding": "gzip, deflate", - "host": "raw.githubusercontent.com", - "user-agent": "https://github.com/renovatebot/renovate", - }, - "method": "GET", - "url": "https://raw.githubusercontent.com/rust-lang/crates.io-index/master/in/va/invalid-crate-name", - }, -] -`; - exports[`datasource/crate getReleases throws for 5xx 1`] = `[Error: external-host-error]`; exports[`datasource/crate getReleases throws for 5xx 2`] = ` diff --git a/lib/datasource/crate/index.spec.ts b/lib/datasource/crate/index.spec.ts index 0821fb55b7b671..5872d9aad32462 100644 --- a/lib/datasource/crate/index.spec.ts +++ b/lib/datasource/crate/index.spec.ts @@ -9,10 +9,6 @@ const res2 = fs.readFileSync( 'lib/datasource/crate/__fixtures__/amethyst', 'utf8' ); -const res3 = fs.readFileSync( - 'lib/datasource/crate/__fixtures__/invalid_crate_data', - 'utf8' -); const baseUrl = 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; @@ -100,23 +96,5 @@ describe('datasource/crate', () => { expect(res).toBeDefined(); expect(httpMock.getTrace()).toMatchSnapshot(); }); - it('returns null if crate name is invalid', async () => { - httpMock.scope(baseUrl).get('/in/va/invalid-crate-name').reply(200, res2); - const res = await getPkgReleases({ - datasource, - depName: 'invalid-crate-name', - }); - expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); - }); - it('returns null for invalid crate data', async () => { - httpMock.scope(baseUrl).get('/so/me/some_crate').reply(200, res3); - const res = await getPkgReleases({ - datasource, - depName: 'some_crate', - }); - expect(res).toBeNull(); - expect(httpMock.getTrace()).toMatchSnapshot(); - }); }); }); diff --git a/lib/datasource/crate/index.ts b/lib/datasource/crate/index.ts index 13d2c000d23652..e6185ed3009a5f 100644 --- a/lib/datasource/crate/index.ts +++ b/lib/datasource/crate/index.ts @@ -40,54 +40,28 @@ export async function getReleases({ 'https://raw.githubusercontent.com/rust-lang/crates.io-index/master/'; const crateUrl = baseUrl + path; try { - let res: any = await http.get(crateUrl); - if (!res || !res.body) { - logger.warn( - { dependency: lookupName }, - `Received invalid crate data from ${crateUrl}` - ); - return null; - } - res = res.body; - res = res.split('\n'); - res = res.map((line) => line.trim()).filter((line) => line.length !== 0); - if (res.length === 0) { - logger.warn( - { dependency: lookupName }, - `Received empty list from ${crateUrl}` - ); - return null; - } - // Filter empty lines (takes care of trailing \n) - // eslint-disable-next-line @typescript-eslint/unbound-method - res = res.map(JSON.parse); - if (res[0].name !== lookupName) { - logger.warn( - { dependency: lookupName }, - `Received invalid crate name from ${crateUrl}` - ); - return null; - } - if (!res[0].vers) { - logger.warn( - { dependency: lookupName }, - `Recieved invalid data (vers field doesn't exist) from ${crateUrl}` - ); - return null; - } + const lines = (await http.get(crateUrl)).body + .split('\n') // break into lines + .map((line) => line.trim()) // remove whitespace + .filter((line) => line.length !== 0) // remove empty lines + .map((line) => JSON.parse(line)); // parse const result: ReleaseResult = { releases: [], }; - result.releases = res.map((version: { vers: string; yanked: boolean }) => { - const release: Release = { - version: version.vers, - }; - if (version.yanked) { - release.isDeprecated = true; - } - return release; - }); - + result.releases = lines + .map((version: { vers: string; yanked: boolean }) => { + const release: Release = { + version: version.vers, + }; + if (version.yanked) { + release.isDeprecated = true; + } + return release; + }) + .filter((release) => release.version); + if (!result.releases.length) { + return null; + } const cacheMinutes = 10; await globalCache.set(cacheNamespace, cacheKey, result, cacheMinutes); return result;