diff --git a/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json new file mode 100644 index 00000000000000..a5cd75cb8425e8 --- /dev/null +++ b/lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json @@ -0,0 +1,103 @@ +{ + "terraform-provider-google-beta": { + "name": "terraform-provider-google-beta", + "versions": { + "1.19.0": { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "shasums": "terraform-provider-google-beta_1.19.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_1.19.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "darwin", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.19.0_darwin_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_darwin_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "freebsd", + "arch": "386", + "filename": "terraform-provider-google-beta_1.19.0_freebsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.19.0", + "os": "freebsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.19.0_freebsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_amd64.zip" + } + ] + }, + "1.20.0": { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "shasums": "terraform-provider-google-beta_1.20.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_1.20.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "openbsd", + "arch": "386", + "filename": "terraform-provider-google-beta_1.20.0_openbsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "openbsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.20.0_openbsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "1.20.0", + "os": "solaris", + "arch": "amd64", + "filename": "terraform-provider-google-beta_1.20.0_solaris_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_solaris_amd64.zip" + } + ] + }, + "2.0.0": { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "shasums": "terraform-provider-google-beta_2.0.0_SHA256SUMS", + "shasums_signature": "terraform-provider-google-beta_2.0.0_SHA256SUMS.sig", + "builds": [ + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "darwin", + "arch": "amd64", + "filename": "terraform-provider-google-beta_2.0.0_darwin_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_darwin_amd64.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "freebsd", + "arch": "386", + "filename": "terraform-provider-google-beta_2.0.0_freebsd_386.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_386.zip" + }, + { + "name": "terraform-provider-google-beta", + "version": "2.0.0", + "os": "freebsd", + "arch": "amd64", + "filename": "terraform-provider-google-beta_2.0.0_freebsd_amd64.zip", + "url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_amd64.zip" + } + ] + } + } + } +} diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap index d52991debb8f1f..577c59d31c1a48 100644 --- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap @@ -1,5 +1,48 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`datasource/terraform getReleases processes data with alternative backend 1`] = ` +Object { + "name": "hashicorp/google-beta", + "releases": Array [ + Object { + "version": "1.19.0", + }, + Object { + "version": "1.20.0", + }, + Object { + "version": "2.0.0", + }, + ], + "versions": Object {}, +} +`; + +exports[`datasource/terraform getReleases processes data with alternative backend 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/v1/providers/hashicorp/google-beta", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, +] +`; + exports[`datasource/terraform getReleases processes real data 1`] = ` Object { "homepage": "https://registry.terraform.io/providers/hashicorp/azurerm", @@ -233,6 +276,16 @@ Array [ "method": "GET", "url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm", }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, ] `; @@ -248,6 +301,16 @@ Array [ "method": "GET", "url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm", }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, ] `; @@ -263,5 +326,17 @@ Array [ "method": "GET", "url": "https://registry.terraform.io/v1/providers/hashicorp/azurerm", }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, ] `; + +exports[`datasource/terraform getReleases simulate failing secondary release source 1`] = `null`; diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts index 4456859da2a7d8..e4e37cc9b1f578 100644 --- a/lib/datasource/terraform-provider/index.spec.ts +++ b/lib/datasource/terraform-provider/index.spec.ts @@ -1,13 +1,17 @@ import fs from 'fs'; import { getPkgReleases } from '..'; import * as httpMock from '../../../test/httpMock'; -import { id as datasource } from '.'; +import { id as datasource, defaultRegistryUrls } from '.'; const consulData: any = fs.readFileSync( 'lib/datasource/terraform-provider/__fixtures__/azurerm-provider.json' ); +const hashicorpReleases: any = fs.readFileSync( + 'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json' +); -const baseUrl = 'https://registry.terraform.io/'; +const primaryUrl = defaultRegistryUrls[0]; +const secondaryUrl = defaultRegistryUrls[1]; describe('datasource/terraform', () => { describe('getReleases', () => { @@ -22,9 +26,10 @@ describe('datasource/terraform', () => { it('returns null for empty result', async () => { httpMock - .scope(baseUrl) + .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') .reply(200, {}); + httpMock.scope(secondaryUrl).get('/index.json').reply(200, {}); expect( await getPkgReleases({ datasource, @@ -34,7 +39,11 @@ describe('datasource/terraform', () => { expect(httpMock.getTrace()).toMatchSnapshot(); }); it('returns null for 404', async () => { - httpMock.scope(baseUrl).get('/v1/providers/hashicorp/azurerm').reply(404); + httpMock + .scope(primaryUrl) + .get('/v1/providers/hashicorp/azurerm') + .reply(404); + httpMock.scope(secondaryUrl).get('/index.json').reply(404); expect( await getPkgReleases({ datasource, @@ -45,9 +54,10 @@ describe('datasource/terraform', () => { }); it('returns null for unknown error', async () => { httpMock - .scope(baseUrl) + .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') .replyWithError(''); + httpMock.scope(secondaryUrl).get('/index.json').replyWithError(''); expect( await getPkgReleases({ datasource, @@ -58,7 +68,7 @@ describe('datasource/terraform', () => { }); it('processes real data', async () => { httpMock - .scope(baseUrl) + .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') .reply(200, JSON.parse(consulData)); const res = await getPkgReleases({ @@ -69,5 +79,41 @@ describe('datasource/terraform', () => { expect(res).not.toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('processes data with alternative backend', async () => { + httpMock + .scope(primaryUrl) + .get('/v1/providers/hashicorp/google-beta') + .reply(404, { + errors: ['Not Found'], + }); + httpMock + .scope(secondaryUrl) + .get('/index.json') + .reply(200, JSON.parse(hashicorpReleases)); + + const res = await getPkgReleases({ + datasource, + depName: 'google-beta', + }); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + it('simulate failing secondary release source', async () => { + httpMock + .scope(primaryUrl) + .get('/v1/providers/hashicorp/google-beta') + .reply(404, { + errors: ['Not Found'], + }); + httpMock.scope(secondaryUrl).get('/index.json').reply(404); + + const res = await getPkgReleases({ + datasource, + depName: 'datadog', + }); + expect(res).toMatchSnapshot(); + expect(res).toBeNull(); + }); }); }); diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index f818251dad686e..5ba096a98e3388 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -1,9 +1,14 @@ +import URL from 'url'; import { logger } from '../../logger'; -import * as globalCache from '../../util/cache/global'; import { Http } from '../../util/http'; -import { DatasourceError, GetReleasesConfig, ReleaseResult } from '../common'; +import { GetReleasesConfig, ReleaseResult } from '../common'; export const id = 'terraform-provider'; +export const defaultRegistryUrls = [ + 'https://registry.terraform.io', + 'https://releases.hashicorp.com', +]; +export const registryStrategy = 'hunt'; const http = new Http(id); @@ -15,30 +20,25 @@ interface TerraformProvider { versions: string[]; } -/** - * terraform-provider.getReleases - * - * This function will fetch a provider from the public Terraform registry and return all semver versions. - */ -export async function getReleases({ - lookupName, -}: GetReleasesConfig): Promise { - const repository = `hashicorp/${lookupName}`; +interface TerraformProviderReleaseBackend { + [key: string]: { + name: string; + versions: VersionsReleaseBackend; + }; +} - logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); - const cacheNamespace = 'terraform-providers'; - const pkgUrl = `https://registry.terraform.io/v1/providers/${repository}`; - const cachedResult = await globalCache.get( - cacheNamespace, - pkgUrl - ); - // istanbul ignore if - if (cachedResult) { - return cachedResult; - } +interface VersionsReleaseBackend { + [key: string]: Record; +} + +async function queryRegistry( + lookupName: string, + registryURL: string, + repository: string +): Promise { try { - const res = (await http.getJson(pkgUrl)).body; - // Simplify response before caching and returning + const backendURL = `${registryURL}/v1/providers/${repository}`; + const res = (await http.getJson(backendURL)).body; const dep: ReleaseResult = { name: repository, versions: {}, @@ -50,33 +50,74 @@ export async function getReleases({ dep.releases = res.versions.map((version) => ({ version, })); - if (pkgUrl.startsWith('https://registry.terraform.io/')) { - dep.homepage = `https://registry.terraform.io/providers/${repository}`; - } + dep.homepage = `${registryURL}/providers/${repository}`; logger.trace({ dep }, 'dep'); - const cacheMinutes = 30; - await globalCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { - if (err.statusCode === 404 || err.code === 'ENOTFOUND') { - logger.debug( - { lookupName }, - `Terraform registry lookup failure: not found` - ); - logger.debug({ - err, - }); - return null; - } - const failureCodes = ['EAI_AGAIN']; - // istanbul ignore if - if (failureCodes.includes(err.code)) { - throw new DatasourceError(err); - } - logger.warn( - { err, lookupName }, - 'Terraform registry failure: Unknown error' + logger.debug( + { lookupName }, + `Terraform registry ("${registryURL}") lookup failure: not found` + ); + logger.debug({ + err, + }); + return null; + } +} + +async function queryReleaseBackend( + lookupName: string, + registryURL: string, + repository: string +): Promise { + const backendLookUpName = `terraform-provider-${lookupName}`; + const backendURL = registryURL + `/index.json`; + try { + const res = ( + await http.getJson(backendURL) + ).body; + const dep: ReleaseResult = { + name: repository, + versions: {}, + releases: null, + }; + dep.releases = Object.keys(res[backendLookUpName].versions).map( + (version) => ({ + version, + }) + ); + logger.trace({ dep }, 'dep'); + return dep; + } catch (err) { + logger.debug( + { lookupName }, + `Terraform registry ("${registryURL}") lookup failure: not found` ); + logger.debug({ + err, + }); return null; } } + +/** + * terraform-provider.getReleases + * + * This function will fetch a provider from the public Terraform registry and return all semver versions. + */ +export async function getReleases({ + lookupName, + registryUrl, +}: GetReleasesConfig): Promise { + const repository = `hashicorp/${lookupName}`; + + logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); + let dep: ReleaseResult = null; + const registryHost = URL.parse(registryUrl).host; + if (registryHost === 'registry.terraform.io') { + dep = await queryRegistry(lookupName, registryUrl, repository); + } else if (registryHost === 'releases.hashicorp.com') { + dep = await queryReleaseBackend(lookupName, registryUrl, repository); + } + return dep; +}