From 961fdaf98a71ef61a3a4dbc9acae12797eedc51a Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Thu, 18 Jun 2020 22:23:53 +0200 Subject: [PATCH] feat(terraform-provider): implement secondary release source --- .../__fixtures__/releaseBackendIndex.json | 103 +++++++++++++++ .../__snapshots__/index.spec.ts.snap | 43 +++++++ .../terraform-provider/index.spec.ts | 22 ++++ lib/datasource/terraform-provider/index.ts | 119 +++++++++++++----- 4 files changed, 258 insertions(+), 29 deletions(-) create mode 100644 lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json diff --git a/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json new file mode 100644 index 00000000000000..a5cd75cb8425e8 --- /dev/null +++ b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json @@ -0,0 +1,103 @@ +{ + "terraform-provider-google-beta": { + "name": "terraform-provider-google-beta", + "versions": { + "1.19.0": { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "shasums": "terraform-provider-google-beta_1.19.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_1.19.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "darwin", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.19.0_darwin_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_darwin_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "freebsd", + "arch": "386", + "filename": "terraform-provider-google-beta_1.19.0_freebsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "freebsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.19.0_freebsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_amd64.zip" + } + ] + }, + "1.20.0": { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "shasums": "terraform-provider-google-beta_1.20.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_1.20.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "openbsd", + "arch": "386", + "filename": "terraform-provider-google-beta_1.20.0_openbsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "openbsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.20.0_openbsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "solaris", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.20.0_solaris_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_solaris_amd64.zip" + } + ] + }, + "2.0.0": { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "shasums": "terraform-provider-google-beta_2.0.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_2.0.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "darwin", + "arch": "amd64", + "filename": "terraform-provider-google-beta_2.0.0_darwin_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_darwin_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "freebsd", + "arch": "386", + "filename": "terraform-provider-google-beta_2.0.0_freebsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "freebsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_2.0.0_freebsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_amd64.zip" + } + ] + } + } + } +} diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap index d52991debb8f1f..0cd047aa237a95 100644 --- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap @@ -1,5 +1,48 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`datasource/terraform getReleases processes data with alternative backend 1`] = ` +Object { + "name": "hashicorp/google-beta", + "releases": Array [ + Object { + "version": "1.19.0", + }, + Object { + "version": "1.20.0", + }, + Object { + "version": "2.0.0", + }, + ], + "versions": Object {}, +} +`; + +exports[`datasource/terraform getReleases processes data with alternative backend 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/v1/providers/hashicorp/google-beta", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, +] +`; + exports[`datasource/terraform getReleases processes real data 1`] = ` Object { "homepage": "https://registry.terraform.io/providers/hashicorp/azurerm", diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts index 4456859da2a7d8..98379443d63a2d 100644 --- a/lib/datasource/terraform-provider/index.spec.ts +++ b/lib/datasource/terraform-provider/index.spec.ts @@ -6,6 +6,9 @@ import { id as datasource } from '.'; const consulData: any = fs.readFileSync( 'lib/datasource/terraform-provider/__fixtures__/azurerm-provider.json' ); +const hashicorpReleases: any = fs.readFileSync( + 'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json' +); const baseUrl = 'https://registry.terraform.io/'; @@ -69,5 +72,24 @@ describe('datasource/terraform', () => { expect(res).not.toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('processes data with alternative backend', async () => { + httpMock + .scope('https://registry.terraform.io') + .get('/v1/providers/hashicorp/google-beta') + .reply(404, { + errors: ['Not Found'], + }); + httpMock + .scope('https://releases.hashicorp.com') + .get('/index.json') + .reply(200, JSON.parse(hashicorpReleases)); + + const res = await terraformProvider.getReleases({ + lookupName: 'google-beta', + }); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); }); }); diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index abe32ad04aa3a4..93d7b1c40e6ab8 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -15,6 +15,84 @@ interface TerraformProvider { versions: string[]; } +interface TerraformProviderReleaseBackend { + [key: string]: { + name: string; + versions: VersionsReleaseBackend; + }; +} + +interface VersionsReleaseBackend { + [key: string]: Record; +} + +async function queryRegistry( + lookupName: string, + backendURL: string, + repository: string +): Promise { + try { + const res = (await http.getJson(backendURL)).body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + if (res.source) { + dep.sourceUrl = res.source; + } + dep.releases = res.versions.map((version) => ({ + version, + })); + dep.homepage = `https://registry.terraform.io/providers/${repository}`; + logger.trace({ dep }, 'dep'); + return dep; + } catch (err) { + logger.debug( + { lookupName }, + `Terraform registry ("registry.terraform.io") lookup failure: not found` + ); + logger.debug({ + err, + }); + return null; + } +} + +async function queryReleaseBackend( + lookupName: string, + backendURL: string, + repository: string +): Promise { + const backendLookUpName = `terraform-provider-${lookupName}`; + try { + const res = ( + await http.getJson(backendURL) + ).body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + dep.releases = Object.keys(res[backendLookUpName].versions).map( + (version) => ({ + version, + }) + ); + logger.trace({ dep }, 'dep'); + return dep; + } catch (err) { + logger.debug( + { lookupName }, + `Terraform registry ("releases.hashicorp.com") lookup failure: not found` + ); + logger.debug({ + err, + }); + return null; + } +} + /** * terraform-provider.getReleases * @@ -26,49 +104,32 @@ export async function getReleases({ }: GetReleasesConfig): Promise { const repository = `hashicorp/${lookupName}`; + const releasesBackendURL = `https://releases.hashicorp.com/index.json`; + const registryBackendURL = `https://registry.terraform.io/v1/providers/${repository}`; + logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); const cacheNamespace = 'terraform-providers'; - const pkgUrl = `https://registry.terraform.io/v1/providers/${repository}`; + const cacheMinutes = 30; const cachedResult = await globalCache.get( cacheNamespace, - pkgUrl + lookupName ); // istanbul ignore if if (cachedResult) { return cachedResult; } try { - const res = (await http.getJson(pkgUrl)).body; - // Simplify response before caching and returning - const dep: ReleaseResult = { - name: repository, - versions: {}, - releases: null, - }; - if (res.source) { - dep.sourceUrl = res.source; + let dep = await queryRegistry(lookupName, registryBackendURL, repository); + if (dep) { + await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); + return dep; } - dep.releases = res.versions.map((version) => ({ - version, - })); - if (pkgUrl.startsWith('https://registry.terraform.io/')) { - dep.homepage = `https://registry.terraform.io/providers/${repository}`; + dep = await queryReleaseBackend(lookupName, releasesBackendURL, repository); + if (dep) { + await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes); } - logger.trace({ dep }, 'dep'); - const cacheMinutes = 30; - await globalCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug( - { lookupName }, - `Terraform registry lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } const failureCodes = ['EAI_AGAIN']; // istanbul ignore if if (failureCodes.includes(err.code)) {