From c4582ff3d732b9e38928a3ce27f1d1b3bf8dac9d Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Thu, 18 Jun 2020 22:32:20 +0200 Subject: [PATCH 1/6] feat(terraform-provider): implement secondary release source --- .../__fixtures__/releaseBackendIndex.json | 103 +++++++++++++++ .../__snapshots__/index.spec.ts.snap | 43 +++++++ .../terraform-provider/index.spec.ts | 23 ++++ lib/datasource/terraform-provider/index.ts | 119 +++++++++++++----- 4 files changed, 259 insertions(+), 29 deletions(-) create mode 100644 lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json diff --git a/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json new file mode 100644 index 00000000000000..a5cd75cb8425e8 --- /dev/null +++ b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json @@ -0,0 +1,103 @@ +{ + "terraform-provider-google-beta": { + "name": "terraform-provider-google-beta", + "versions": { + "1.19.0": { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "shasums": "terraform-provider-google-beta_1.19.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_1.19.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "darwin", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.19.0_darwin_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_darwin_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "freebsd", + "arch": "386", + "filename": "terraform-provider-google-beta_1.19.0_freebsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "freebsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.19.0_freebsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_amd64.zip" + } + ] + }, + "1.20.0": { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "shasums": "terraform-provider-google-beta_1.20.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_1.20.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "openbsd", + "arch": "386", + "filename": "terraform-provider-google-beta_1.20.0_openbsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "openbsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.20.0_openbsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "solaris", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.20.0_solaris_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_solaris_amd64.zip" + } + ] + }, + "2.0.0": { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "shasums": "terraform-provider-google-beta_2.0.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_2.0.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "darwin", + "arch": "amd64", + "filename": "terraform-provider-google-beta_2.0.0_darwin_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_darwin_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "freebsd", + "arch": "386", + "filename": "terraform-provider-google-beta_2.0.0_freebsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "freebsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_2.0.0_freebsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_amd64.zip" + } + ] + } + } + } +} diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap index d52991debb8f1f..0cd047aa237a95 100644 --- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap @@ -1,5 +1,48 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`datasource/terraform getReleases processes data with alternative backend 1`] = ` +Object { + "name": "hashicorp/google-beta", + "releases": Array [ + Object { + "version": "1.19.0", + }, + Object { + "version": "1.20.0", + }, + Object { + "version": "2.0.0", + }, + ], + "versions": Object {}, +} +`; + +exports[`datasource/terraform getReleases processes data with alternative backend 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/v1/providers/hashicorp/google-beta", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, +] +`; + exports[`datasource/terraform getReleases processes real data 1`] = ` Object { "homepage": "https://registry.terraform.io/providers/hashicorp/azurerm", diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts index 4456859da2a7d8..09b0a77e4a7cb6 100644 --- a/lib/datasource/terraform-provider/index.spec.ts +++ b/lib/datasource/terraform-provider/index.spec.ts @@ -6,6 +6,9 @@ import { id as datasource } from '.'; const consulData: any = fs.readFileSync( 'lib/datasource/terraform-provider/__fixtures__/azurerm-provider.json' ); +const hashicorpReleases: any = fs.readFileSync( + 'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json' +); const baseUrl = 'https://registry.terraform.io/'; @@ -69,5 +72,25 @@ describe('datasource/terraform', () => { expect(res).not.toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('processes data with alternative backend', async () => { + httpMock + .scope('https://registry.terraform.io') + .get('/v1/providers/hashicorp/google-beta') + .reply(404, { + errors: ['Not Found'], + }); + httpMock + .scope('https://releases.hashicorp.com') + .get('/index.json') + .reply(200, JSON.parse(hashicorpReleases)); + + const res = await getPkgReleases({ + datasource, + depName: 'google-beta', + }); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); }); }); diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index abe32ad04aa3a4..93d7b1c40e6ab8 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -15,6 +15,84 @@ interface TerraformProvider { versions: string[]; } +interface TerraformProviderReleaseBackend { + [key: string]: { + name: string; + versions: VersionsReleaseBackend; + }; +} + +interface VersionsReleaseBackend { + [key: string]: Record; +} + +async function queryRegistry( + lookupName: string, + backendURL: string, + repository: string +): Promise { + try { + const res = (await http.getJson(backendURL)).body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + if (res.source) { + dep.sourceUrl = res.source; + } + dep.releases = res.versions.map((version) => ({ + version, + })); + dep.homepage = `https://registry.terraform.io/providers/${repository}`; + logger.trace({ dep }, 'dep'); + return dep; + } catch (err) { + logger.debug( + { lookupName }, + `Terraform registry ("registry.terraform.io") lookup failure: not found` + ); + logger.debug({ + err, + }); + return null; + } +} + +async function queryReleaseBackend( + lookupName: string, + backendURL: string, + repository: string +): Promise { + const backendLookUpName = `terraform-provider-${lookupName}`; + try { + const res = ( + await http.getJson(backendURL) + ).body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + dep.releases = Object.keys(res[backendLookUpName].versions).map( + (version) => ({ + version, + }) + ); + logger.trace({ dep }, 'dep'); + return dep; + } catch (err) { + logger.debug( + { lookupName }, + `Terraform registry ("releases.hashicorp.com") lookup failure: not found` + ); + logger.debug({ + err, + }); + return null; + } +} + /** * terraform-provider.getReleases * @@ -26,49 +104,32 @@ export async function getReleases({ }: GetReleasesConfig): Promise { const repository = `hashicorp/${lookupName}`; + const releasesBackendURL = `https://releases.hashicorp.com/index.json`; + const registryBackendURL = `https://registry.terraform.io/v1/providers/${repository}`; + logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); const cacheNamespace = 'terraform-providers'; - const pkgUrl = `https://registry.terraform.io/v1/providers/${repository}`; + const cacheMinutes = 30; const cachedResult = await globalCache.get( cacheNamespace, - pkgUrl + lookupName ); // istanbul ignore if if (cachedResult) { return cachedResult; } try { - const res = (await http.getJson(pkgUrl)).body; - // Simplify response before caching and returning - const dep: ReleaseResult = { - name: repository, - versions: {}, - releases: null, - }; - if (res.source) { - dep.sourceUrl = res.source; + let dep = await queryRegistry(lookupName, registryBackendURL, repository); + if (dep) { + await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); + return dep; } - dep.releases = res.versions.map((version) => ({ - version, - })); - if (pkgUrl.startsWith('https://registry.terraform.io/')) { - dep.homepage = `https://registry.terraform.io/providers/${repository}`; + dep = await queryReleaseBackend(lookupName, releasesBackendURL, repository); + if (dep) { + await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); } - logger.trace({ dep }, 'dep'); - const cacheMinutes = 30; - await globalCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug( - { lookupName }, - `Terraform registry lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } const failureCodes = ['EAI_AGAIN']; // istanbul ignore if if (failureCodes.includes(err.code)) { From e0dd5eb82c875d74768d13bd8e941553ce1ca9dd Mon Sep 17 00:00:00 2001 From: Jamie Magee Date: Thu, 18 Jun 2020 23:09:34 +0200 Subject: [PATCH 2/6] fix mocks --- .../__snapshots__/index.spec.ts.snap | 30 +++++++++++++++++++ .../terraform-provider/index.spec.ts | 21 ++++++++----- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap index 0cd047aa237a95..05a5120c63e474 100644 --- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap @@ -276,6 +276,16 @@ Array [ "method": "GET", "url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm", }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, ] `; @@ -291,6 +301,16 @@ Array [ "method": "GET", "url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm", }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, ] `; @@ -306,5 +326,15 @@ Array [ "method": "GET", "url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm", }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, ] `; diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts index 09b0a77e4a7cb6..b8a2e48ecb747a 100644 --- a/lib/datasource/terraform-provider/index.spec.ts +++ b/lib/datasource/terraform-provider/index.spec.ts @@ -10,7 +10,8 @@ const hashicorpReleases: any = fs.readFileSync( 'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json' ); -const baseUrl = 'https://registry.terraform.io/'; +const primaryUrl = 'https://registry.terraform.io/'; +const secondaryUrl = 'https://releases.hashicorp.com/'; describe('datasource/terraform', () => { describe('getReleases', () => { @@ -25,9 +26,10 @@ describe('datasource/terraform', () => { it('returns null for empty result', async () => { httpMock - .scope(baseUrl) + .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') .reply(200, {}); + httpMock.scope(secondaryUrl).get('/index.json').reply(200, {}); expect( await getPkgReleases({ datasource, @@ -37,7 +39,11 @@ describe('datasource/terraform', () => { expect(httpMock.getTrace()).toMatchSnapshot(); }); it('returns null for 404', async () => { - httpMock.scope(baseUrl).get('/v1/providers/hashicorp/azurerm').reply(404); + httpMock + .scope(primaryUrl) + .get('/v1/providers/hashicorp/azurerm') + .reply(404); + httpMock.scope(secondaryUrl).get('/index.json').reply(404); expect( await getPkgReleases({ datasource, @@ -48,9 +54,10 @@ describe('datasource/terraform', () => { }); it('returns null for unknown error', async () => { httpMock - .scope(baseUrl) + .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') .replyWithError(''); + httpMock.scope(secondaryUrl).get('/index.json').replyWithError(''); expect( await getPkgReleases({ datasource, @@ -61,7 +68,7 @@ describe('datasource/terraform', () => { }); it('processes real data', async () => { httpMock - .scope(baseUrl) + .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') .reply(200, JSON.parse(consulData)); const res = await getPkgReleases({ @@ -74,13 +81,13 @@ describe('datasource/terraform', () => { }); it('processes data with alternative backend', async () => { httpMock - .scope('https://registry.terraform.io') + .scope(primaryUrl) .get('/v1/providers/hashicorp/google-beta') .reply(404, { errors: ['Not Found'], }); httpMock - .scope('https://releases.hashicorp.com') + .scope(secondaryUrl) .get('/index.json') .reply(200, JSON.parse(hashicorpReleases)); From ba53c4dee4b8858ece3804c87c013d19817e2a7b Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Fri, 19 Jun 2020 09:19:11 +0200 Subject: [PATCH 3/6] adding unit-test, remove unnecessary catch clause --- .../__snapshots__/index.spec.ts.snap | 2 ++ .../terraform-provider/index.spec.ts | 16 ++++++++++ lib/datasource/terraform-provider/index.ts | 31 ++++++------------- 3 files changed, 27 insertions(+), 22 deletions(-) diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap index 05a5120c63e474..577c59d31c1a48 100644 --- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap @@ -338,3 +338,5 @@ Array [ }, ] `; + +exports[`datasource/terraform getReleases simulate failing secondary release source 1`] = `null`; diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts index b8a2e48ecb747a..0ffce7fa7a8fc6 100644 --- a/lib/datasource/terraform-provider/index.spec.ts +++ b/lib/datasource/terraform-provider/index.spec.ts @@ -99,5 +99,21 @@ describe('datasource/terraform', () => { expect(res).not.toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('simulate failing secondary release source', async () => { + httpMock + .scope(primaryUrl) + .get('/v1/providers/hashicorp/google-beta') + .reply(404, { + errors: ['Not Found'], + }); + httpMock.scope(secondaryUrl).get('/index.json').reply(404); + + const res = await getPkgReleases({ + datasource, + depName: 'datadog', + }); + expect(res).toMatchSnapshot(); + expect(res).toBeNull(); + }); }); }); diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index 93d7b1c40e6ab8..508faafb6ef37e 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -1,7 +1,7 @@ import { logger } from '../../logger'; import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'terraform-provider'; @@ -118,27 +118,14 @@ export async function getReleases({ if (cachedResult) { return cachedResult; } - try { - let dep = await queryRegistry(lookupName, registryBackendURL, repository); - if (dep) { - await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); - return dep; - } - dep = await queryReleaseBackend(lookupName, releasesBackendURL, repository); - if (dep) { - await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); - } + let dep = await queryRegistry(lookupName, registryBackendURL, repository); + if (dep) { + await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); return dep; - } catch (err) { - const failureCodes = ['EAI_AGAIN']; - // istanbul ignore if - if (failureCodes.includes(err.code)) { - throw new DatasourceError(err); - } - logger.warn( - { err, lookupName }, - 'Terraform registry failure: Unknown error' - ); - return null; } + dep = await queryReleaseBackend(lookupName, releasesBackendURL, repository); + if (dep) { + await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); + } + return dep; } From d8dfea26a74a593348f9a6f0b174390b61015f41 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 20 Jun 2020 13:31:39 +0200 Subject: [PATCH 4/6] chore(terraform-provider): move to registryStrategy and global cache --- .../terraform-provider/index.spec.ts | 6 +-- lib/datasource/terraform-provider/index.ts | 45 ++++++++----------- 2 files changed, 21 insertions(+), 30 deletions(-) diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts index 0ffce7fa7a8fc6..e4e37cc9b1f578 100644 --- a/lib/datasource/terraform-provider/index.spec.ts +++ b/lib/datasource/terraform-provider/index.spec.ts @@ -1,7 +1,7 @@ import fs from 'fs'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; -import { id as datasource } from '.'; +import { id as datasource, defaultRegistryUrls } from '.'; const consulData: any = fs.readFileSync( 'lib/datasource/terraform-provider/__fixtures__/azurerm-provider.json' @@ -10,8 +10,8 @@ const hashicorpReleases: any = fs.readFileSync( 'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json' ); -const primaryUrl = 'https://registry.terraform.io/'; -const secondaryUrl = 'https://releases.hashicorp.com/'; +const primaryUrl = defaultRegistryUrls[0]; +const secondaryUrl = defaultRegistryUrls[1]; describe('datasource/terraform', () => { describe('getReleases', () => { diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index 5e0a99df34285b..f53e7058b0256b 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -1,9 +1,13 @@ import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'terraform-provider'; +export const defaultRegistryUrls = [ + 'https://registry.terraform.io', + 'https://releases.hashicorp.com', +]; +export const registryStrategy = 'hunt'; const http = new Http(id); @@ -28,10 +32,11 @@ interface VersionsReleaseBackend { async function queryRegistry( lookupName: string, - backendURL: string, + registryURL: string, repository: string ): Promise { try { + const backendURL = `${registryURL}/v1/providers/${repository}`; const res = (await http.getJson(backendURL)).body; const dep: ReleaseResult = { name: repository, @@ -44,13 +49,13 @@ async function queryRegistry( dep.releases = res.versions.map((version) => ({ version, })); - dep.homepage = `https://registry.terraform.io/providers/${repository}`; + dep.homepage = `${registryURL}/providers/${repository}`; logger.trace({ dep }, 'dep'); return dep; } catch (err) { logger.debug( { lookupName }, - `Terraform registry ("registry.terraform.io") lookup failure: not found` + `Terraform registry ("${registryURL}") lookup failure: not found` ); logger.debug({ err, @@ -61,10 +66,11 @@ async function queryRegistry( async function queryReleaseBackend( lookupName: string, - backendURL: string, + registryURL: string, repository: string ): Promise { const backendLookUpName = `terraform-provider-${lookupName}`; + const backendURL = registryURL + `/index.json`; try { const res = ( await http.getJson(backendURL) @@ -84,7 +90,7 @@ async function queryReleaseBackend( } catch (err) { logger.debug( { lookupName }, - `Terraform registry ("releases.hashicorp.com") lookup failure: not found` + `Terraform registry ("${registryURL}") lookup failure: not found` ); logger.debug({ err, @@ -100,31 +106,16 @@ async function queryReleaseBackend( */ export async function getReleases({ lookupName, + registryUrl, }: GetReleasesConfig): Promise { const repository = `hashicorp/${lookupName}`; - const releasesBackendURL = `https://releases.hashicorp.com/index.json`; - const registryBackendURL = `https://registry.terraform.io/v1/providers/${repository}`; - logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); - const cacheNamespace = 'terraform-providers'; - const cacheMinutes = 30; - const cachedResult = await globalCache.get( - cacheNamespace, - lookupName - ); - // istanbul ignore if - if (cachedResult) { - return cachedResult; - } - let dep = await queryRegistry(lookupName, registryBackendURL, repository); - if (dep) { - await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); - return dep; - } - dep = await queryReleaseBackend(lookupName, releasesBackendURL, repository); - if (dep) { - await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); + let dep: ReleaseResult = null; + if (registryUrl.includes('registry.terraform.io')) { + dep = await queryRegistry(lookupName, registryUrl, repository); + } else if (registryUrl.includes('releases.hashicorp.com')) { + dep = await queryReleaseBackend(lookupName, registryUrl, repository); } return dep; } From cdf1516f9abf5d75307f425665c0fb1003598586 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 20 Jun 2020 13:41:49 +0200 Subject: [PATCH 5/6] fix(terraform-provider): LGTM error: not sanitized URL --- lib/datasource/terraform-provider/index.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index f53e7058b0256b..eacbb1658ff5db 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -112,9 +112,9 @@ export async function getReleases({ logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); let dep: ReleaseResult = null; - if (registryUrl.includes('registry.terraform.io')) { + if (registryUrl.startsWith('https://registry.terraform.io')) { dep = await queryRegistry(lookupName, registryUrl, repository); - } else if (registryUrl.includes('releases.hashicorp.com')) { + } else if (registryUrl.includes('https://releases.hashicorp.com')) { dep = await queryReleaseBackend(lookupName, registryUrl, repository); } return dep; From 3aeba62b5b260ca6b50084e35b18ac7a56723529 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 20 Jun 2020 14:09:51 +0200 Subject: [PATCH 6/6] fixup!(terraform-provider): LGTM error: not sanitized URL --- lib/datasource/terraform-provider/index.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index eacbb1658ff5db..5ba096a98e3388 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -1,3 +1,4 @@ +import URL from 'url'; import { logger } from '../../logger'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -112,9 +113,10 @@ export async function getReleases({ logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); let dep: ReleaseResult = null; - if (registryUrl.startsWith('https://registry.terraform.io')) { + const registryHost = URL.parse(registryUrl).host; + if (registryHost === 'registry.terraform.io') { dep = await queryRegistry(lookupName, registryUrl, repository); - } else if (registryUrl.includes('https://releases.hashicorp.com')) { + } else if (registryHost === 'releases.hashicorp.com') { dep = await queryReleaseBackend(lookupName, registryUrl, repository); } return dep;