From 937306452df3aca8f917d452ec7c96bef58b8764 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Thu, 9 Jul 2020 21:06:02 +0200 Subject: [PATCH 1/8] chore(terraform-provider): removing datasource caching in favor of global cache --- lib/datasource/terraform-module/index.ts | 13 ------------- 1 file changed, 13 deletions(-) diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 50c7a7b90d016e..5e5a79806c7787 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -1,6 +1,5 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; -import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -64,16 +63,6 @@ export async function getReleases({ { registry, terraformRepository: repository }, 'terraform.getDependencies()' ); - const cacheNamespace = 'terraform-module'; - const pkgUrl = `${registry}/v1/modules/${repository}`; - const cachedResult = await packageCache.get( - cacheNamespace, - pkgUrl - ); - // istanbul ignore if - if (cachedResult) { - return cachedResult; - } try { const res = (await http.getJson(pkgUrl)).body; const returnedName = res.namespace + '/' + res.name + '/' + res.provider; @@ -97,8 +86,6 @@ export async function getReleases({ dep.homepage = `https://registry.terraform.io/modules/${repository}`; } logger.trace({ dep }, 'dep'); - const cacheMinutes = 30; - await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { const failureCodes = ['EAI_AGAIN']; From 0ea077e3237a9e80bfeed965bfa881c119072cd4 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Thu, 9 Jul 2020 21:31:10 +0200 Subject: [PATCH 2/8] feat(terraform-provider): use terraform service discovery for modules --- .../service-custom-discovery.json | 8 + .../__fixtures__/service-discovery.json | 4 + .../__snapshots__/index.spec.ts.snap | 180 ++++++++++++++++++ lib/datasource/terraform-module/index.spec.ts | 58 +++++- lib/datasource/terraform-module/index.ts | 11 ++ 5 files changed, 255 insertions(+), 6 deletions(-) create mode 100644 lib/datasource/terraform-module/__fixtures__/service-custom-discovery.json create mode 100644 lib/datasource/terraform-module/__fixtures__/service-discovery.json diff --git a/lib/datasource/terraform-module/__fixtures__/service-custom-discovery.json b/lib/datasource/terraform-module/__fixtures__/service-custom-discovery.json new file mode 100644 index 00000000000000..6b3cbb2141db6b --- /dev/null +++ b/lib/datasource/terraform-module/__fixtures__/service-custom-discovery.json @@ -0,0 +1,8 @@ +{ + "modules.v1": "/api/registry/v1/modules/", + "state.v2": "/api/v2/", + "tfe.v2": "/api/v2/", + "tfe.v2.1": "/api/v2/", + "tfe.v2.2": "/api/v2/", + "versions.v1": "https://checkpoint-api.hashicorp.com/v1/versions/" +} diff --git a/lib/datasource/terraform-module/__fixtures__/service-discovery.json b/lib/datasource/terraform-module/__fixtures__/service-discovery.json new file mode 100644 index 00000000000000..db3cc145ea3c03 --- /dev/null +++ b/lib/datasource/terraform-module/__fixtures__/service-discovery.json @@ -0,0 +1,4 @@ +{ + "modules.v1": "/v1/modules/", + "providers.v1": "/v1/providers/" +} diff --git a/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap index f325481abea80d..acb9751d9570a9 100644 --- a/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap @@ -82,6 +82,16 @@ Object { exports[`datasource/terraform-module getReleases processes real data 2`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -95,6 +105,111 @@ Array [ ] `; +exports[`datasource/terraform-module getReleases processes real data on changed subpath 1`] = ` +Object { + "homepage": "https://registry.terraform.io/modules/hashicorp/consul/aws", + "name": "hashicorp/consul/aws", + "releases": Array [ + Object { + "version": "0.0.1", + }, + Object { + "version": "0.0.2", + }, + Object { + "version": "0.0.3", + }, + Object { + "version": "0.0.4", + }, + Object { + "version": "0.0.5", + }, + Object { + "version": "0.1.0", + }, + Object { + "version": "0.1.1", + }, + Object { + "version": "0.1.2", + }, + Object { + "version": "0.2.0", + }, + Object { + "version": "0.2.1", + }, + Object { + "version": "0.2.2", + }, + Object { + "version": "0.3.0", + }, + Object { + "version": "0.3.1", + }, + Object { + "version": "0.3.2", + }, + Object { + "version": "0.3.3", + }, + Object { + "version": "0.3.4", + }, + Object { + "version": "0.3.5", + }, + Object { + "version": "0.3.6", + }, + Object { + "version": "0.3.7", + }, + Object { + "version": "0.3.8", + }, + Object { + "version": "0.3.9", + }, + Object { + "version": "0.3.10", + }, + Object { + "version": "0.4.0", + }, + ], + "sourceUrl": "https://github.com/hashicorp/terraform-aws-consul", + "versions": Object {}, +} +`; + +exports[`datasource/terraform-module getReleases processes real data on changed subpath 2`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/api/registry/v1/modules/hashicorp/consul/aws", + }, +] +`; + exports[`datasource/terraform-module getReleases processes with registry in name 1`] = ` Object { "homepage": "https://registry.terraform.io/modules/hashicorp/consul/aws", @@ -177,6 +292,16 @@ Object { exports[`datasource/terraform-module getReleases processes with registry in name 2`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -192,6 +317,16 @@ Array [ exports[`datasource/terraform-module getReleases rejects mismatch 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "terraform.company.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://terraform.company.com/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -205,8 +340,33 @@ Array [ ] `; +exports[`datasource/terraform-module getReleases rejects servicediscovery 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "terraform.company.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://terraform.company.com/.well-known/terraform.json", + }, +] +`; + exports[`datasource/terraform-module getReleases returns null for 404 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -222,6 +382,16 @@ Array [ exports[`datasource/terraform-module getReleases returns null for empty result 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -237,6 +407,16 @@ Array [ exports[`datasource/terraform-module getReleases returns null for unknown error 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", diff --git a/lib/datasource/terraform-module/index.spec.ts b/lib/datasource/terraform-module/index.spec.ts index df5e7c7b6c29fc..544420715b18a4 100644 --- a/lib/datasource/terraform-module/index.spec.ts +++ b/lib/datasource/terraform-module/index.spec.ts @@ -6,6 +6,12 @@ import { id as datasource } from '.'; const consulData: any = fs.readFileSync( 'lib/datasource/terraform-module/__fixtures__/registry-consul.json' ); +const serviceDiscoveryResult: any = fs.readFileSync( + 'lib/datasource/terraform-module/__fixtures__/service-discovery.json' +); +const serviceDiscoveryCustomResult: any = fs.readFileSync( + 'lib/datasource/terraform-module/__fixtures__/service-custom-discovery.json' +); const baseUrl = 'https://registry.terraform.io'; @@ -24,7 +30,9 @@ describe('datasource/terraform-module', () => { httpMock .scope(baseUrl) .get('/v1/modules/hashicorp/consul/aws') - .reply(200, {}); + .reply(200, {}) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); expect( await getPkgReleases({ datasource, @@ -37,7 +45,9 @@ describe('datasource/terraform-module', () => { httpMock .scope(baseUrl) .get('/v1/modules/hashicorp/consul/aws') - .reply(404, {}); + .reply(404, {}) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); expect( await getPkgReleases({ datasource, @@ -50,7 +60,9 @@ describe('datasource/terraform-module', () => { httpMock .scope(baseUrl) .get('/v1/modules/hashicorp/consul/aws') - .replyWithError(''); + .replyWithError('') + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); expect( await getPkgReleases({ datasource, @@ -63,7 +75,9 @@ describe('datasource/terraform-module', () => { httpMock .scope(baseUrl) .get('/v1/modules/hashicorp/consul/aws') - .reply(200, consulData); + .reply(200, consulData) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); const res = await getPkgReleases({ datasource, depName: 'hashicorp/consul/aws', @@ -76,7 +90,9 @@ describe('datasource/terraform-module', () => { httpMock .scope(baseUrl) .get('/v1/modules/hashicorp/consul/aws') - .reply(200, consulData); + .reply(200, consulData) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); const res = await getPkgReleases({ datasource, depName: 'registry.terraform.io/hashicorp/consul/aws', @@ -89,7 +105,22 @@ describe('datasource/terraform-module', () => { httpMock .scope('https://terraform.company.com') .get('/v1/modules/consul/foo') - .reply(200, consulData); + .reply(200, consulData) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); + const res = await getPkgReleases({ + datasource, + depName: 'consul/foo', + registryUrls: ['https://terraform.company.com'], + }); + expect(res).toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); + it('rejects servicediscovery', async () => { + httpMock + .scope('https://terraform.company.com') + .get('/.well-known/terraform.json') + .reply(404); const res = await getPkgReleases({ datasource, depName: 'consul/foo', @@ -98,5 +129,20 @@ describe('datasource/terraform-module', () => { expect(res).toBeNull(); expect(httpMock.getTrace()).toMatchSnapshot(); }); + it('processes real data on changed subpath', async () => { + httpMock + .scope(baseUrl) + .get('/api/registry/v1/modules/hashicorp/consul/aws') + .reply(200, consulData) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryCustomResult); + const res = await getPkgReleases({ + datasource, + depName: 'hashicorp/consul/aws', + }); + expect(res).toMatchSnapshot(); + expect(res).not.toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); }); }); diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 5e5a79806c7787..2286636b910728 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -44,6 +44,10 @@ interface TerraformRelease { versions: string[]; } +interface ServiceDiscoveryResult { + 'modules.v1': string; +} + /** * terraform.getReleases * @@ -64,6 +68,13 @@ export async function getReleases({ 'terraform.getDependencies()' ); try { + const serviceDiscovery = ( + await http.getJson( + `${registryUrl}/.well-known/terraform.json` + ) + ).body; + + const pkgUrl = `${registry}${serviceDiscovery['modules.v1']}${repository}`; const res = (await http.getJson(pkgUrl)).body; const returnedName = res.namespace + '/' + res.name + '/' + res.provider; if (returnedName !== repository) { From 1569f917bc8147fd75e9f176ceddb1f12c2b8290 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Thu, 9 Jul 2020 22:22:34 +0200 Subject: [PATCH 3/8] feat(terraform-provider): use terraform service discovery for provider registry --- lib/datasource/terraform-module/index.ts | 5 +- .../__fixtures__/service-discovery.json | 4 + .../__snapshots__/index.spec.ts.snap | 75 +++++++++++++++++++ .../terraform-provider/index.spec.ts | 38 ++++++++-- lib/datasource/terraform-provider/index.ts | 14 +++- 5 files changed, 124 insertions(+), 12 deletions(-) create mode 100644 lib/datasource/terraform-provider/__fixtures__/service-discovery.json diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index 2286636b910728..b459c30f31708c 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -44,8 +44,9 @@ interface TerraformRelease { versions: string[]; } -interface ServiceDiscoveryResult { - 'modules.v1': string; +export interface ServiceDiscoveryResult { + 'modules.v1'?: string; + 'providers.v1'?: string; } /** diff --git a/lib/datasource/terraform-provider/__fixtures__/service-discovery.json b/lib/datasource/terraform-provider/__fixtures__/service-discovery.json new file mode 100644 index 00000000000000..db3cc145ea3c03 --- /dev/null +++ b/lib/datasource/terraform-provider/__fixtures__/service-discovery.json @@ -0,0 +1,4 @@ +{ + "modules.v1": "/v1/modules/", + "providers.v1": "/v1/providers/" +} diff --git a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap index 577c59d31c1a48..cc8000d12f97d7 100644 --- a/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap @@ -20,6 +20,16 @@ Object { exports[`datasource/terraform getReleases processes data with alternative backend 2`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -251,6 +261,16 @@ Object { exports[`datasource/terraform getReleases processes real data 2`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -266,6 +286,16 @@ Array [ exports[`datasource/terraform getReleases returns null for 404 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -291,6 +321,16 @@ Array [ exports[`datasource/terraform getReleases returns null for empty result 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", @@ -314,8 +354,43 @@ Array [ ] `; +exports[`datasource/terraform getReleases returns null for error in service discovery 1`] = ` +Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "releases.hashicorp.com", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://releases.hashicorp.com/index.json", + }, +] +`; + exports[`datasource/terraform getReleases returns null for unknown error 1`] = ` Array [ + Object { + "headers": Object { + "accept": "application/json", + "accept-encoding": "gzip, deflate", + "host": "registry.terraform.io", + "user-agent": "https://github.com/renovatebot/renovate", + }, + "method": "GET", + "url": "https://registry.terraform.io/.well-known/terraform.json", + }, Object { "headers": Object { "accept": "application/json", diff --git a/lib/datasource/terraform-provider/index.spec.ts b/lib/datasource/terraform-provider/index.spec.ts index e4e37cc9b1f578..53c750c492035a 100644 --- a/lib/datasource/terraform-provider/index.spec.ts +++ b/lib/datasource/terraform-provider/index.spec.ts @@ -9,6 +9,9 @@ const consulData: any = fs.readFileSync( const hashicorpReleases: any = fs.readFileSync( 'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json' ); +const serviceDiscoveryResult: any = fs.readFileSync( + 'lib/datasource/terraform-module/__fixtures__/service-discovery.json' +); const primaryUrl = defaultRegistryUrls[0]; const secondaryUrl = defaultRegistryUrls[1]; @@ -28,7 +31,9 @@ describe('datasource/terraform', () => { httpMock .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') - .reply(200, {}); + .reply(200, {}) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); httpMock.scope(secondaryUrl).get('/index.json').reply(200, {}); expect( await getPkgReleases({ @@ -42,7 +47,9 @@ describe('datasource/terraform', () => { httpMock .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') - .reply(404); + .reply(404) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); httpMock.scope(secondaryUrl).get('/index.json').reply(404); expect( await getPkgReleases({ @@ -56,7 +63,9 @@ describe('datasource/terraform', () => { httpMock .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') - .replyWithError(''); + .replyWithError('') + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); httpMock.scope(secondaryUrl).get('/index.json').replyWithError(''); expect( await getPkgReleases({ @@ -70,7 +79,9 @@ describe('datasource/terraform', () => { httpMock .scope(primaryUrl) .get('/v1/providers/hashicorp/azurerm') - .reply(200, JSON.parse(consulData)); + .reply(200, JSON.parse(consulData)) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); const res = await getPkgReleases({ datasource, depName: 'azurerm', @@ -85,7 +96,9 @@ describe('datasource/terraform', () => { .get('/v1/providers/hashicorp/google-beta') .reply(404, { errors: ['Not Found'], - }); + }) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); httpMock .scope(secondaryUrl) .get('/index.json') @@ -105,7 +118,9 @@ describe('datasource/terraform', () => { .get('/v1/providers/hashicorp/google-beta') .reply(404, { errors: ['Not Found'], - }); + }) + .get('/.well-known/terraform.json') + .reply(200, serviceDiscoveryResult); httpMock.scope(secondaryUrl).get('/index.json').reply(404); const res = await getPkgReleases({ @@ -115,5 +130,16 @@ describe('datasource/terraform', () => { expect(res).toMatchSnapshot(); expect(res).toBeNull(); }); + it('returns null for error in service discovery', async () => { + httpMock.scope(primaryUrl).get('/.well-known/terraform.json').reply(404); + httpMock.scope(secondaryUrl).get('/index.json').replyWithError(''); + expect( + await getPkgReleases({ + datasource, + depName: 'azurerm', + }) + ).toBeNull(); + expect(httpMock.getTrace()).toMatchSnapshot(); + }); }); }); diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index 64be934edfee1b..1a29f21dc1f24e 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -2,6 +2,7 @@ import URL from 'url'; import { logger } from '../../logger'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; +import { ServiceDiscoveryResult } from '../terraform-module'; export const id = 'terraform-provider'; export const defaultRegistryUrls = [ @@ -36,7 +37,12 @@ async function queryRegistry( registryURL: string, repository: string ): Promise { - const backendURL = `${registryURL}/v1/providers/${repository}`; + const serviceDiscovery = ( + await http.getJson( + `${registryURL}/.well-known/terraform.json` + ) + ).body; + const backendURL = `${registryURL}${serviceDiscovery['providers.v1']}${repository}`; const res = (await http.getJson(backendURL)).body; const dep: ReleaseResult = { name: repository, @@ -91,10 +97,10 @@ export async function getReleases({ logger.debug({ lookupName }, 'terraform-provider.getDependencies()'); let dep: ReleaseResult = null; const registryHost = URL.parse(registryUrl).host; - if (registryHost === 'registry.terraform.io') { - dep = await queryRegistry(lookupName, registryUrl, repository); - } else if (registryHost === 'releases.hashicorp.com') { + if (registryHost === 'releases.hashicorp.com') { dep = await queryReleaseBackend(lookupName, registryUrl, repository); + } else { + dep = await queryRegistry(lookupName, registryUrl, repository); } return dep; } From 7f1fee2a84e28542b89bbf92458441482a426450 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Thu, 9 Jul 2020 23:32:48 +0200 Subject: [PATCH 4/8] feat(terraform-provider): extract registry urls for modules --- lib/manager/terraform/__snapshots__/extract.spec.ts.snap | 9 +++++++++ lib/manager/terraform/modules.ts | 5 +++++ 2 files changed, 14 insertions(+) diff --git a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap index 2d3476bb075324..7ef5ccbd7c8493 100644 --- a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap @@ -154,6 +154,9 @@ Object { "depName": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depNameShort": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depType": "terraform", + "registryUrls": Array [ + "app.terraform.io", + ], }, Object { "currentValue": "~> 1.1", @@ -161,6 +164,9 @@ Object { "depName": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depNameShort": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depType": "terraform", + "registryUrls": Array [ + "app.terraform.io", + ], }, Object { "currentValue": "~~ 1.1", @@ -168,6 +174,9 @@ Object { "depName": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depNameShort": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depType": "terraform", + "registryUrls": Array [ + "app.terraform.io", + ], }, Object { "currentValue": ">= 1.0.0, <= 2.0.0", diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts index 9c84b94d0af38e..a60076a1a33584 100644 --- a/lib/manager/terraform/modules.ts +++ b/lib/manager/terraform/modules.ts @@ -10,6 +10,7 @@ import { ExtractionResult, TerraformDependencyTypes } from './util'; const githubRefMatchRegex = /github.com([/:])(?[^/]+\/[a-z0-9-.]+).*\?ref=(?.*)$/; const gitTagsRefMatchRegex = /(?:git::)?(?(?:http|https|ssh):\/\/(?:.*@)?(?.*.*\/(?.*\/.*)))\?ref=(?.*)$/; +const hostnameMatchRegex = /^(http(s)?:\/\/)?(?([\w|\d]+\.)+[\w|\d]+)/; export function extractTerraformModule( startingLine: number, @@ -62,6 +63,10 @@ export function analyseTerraformModule(dep: PackageDependency): void { if (moduleParts[0] === '..') { dep.skipReason = SkipReason.Local; } else if (moduleParts.length >= 3) { + const hostnameMatch = hostnameMatchRegex.exec(dep.managerData.source); + if (hostnameMatch) { + dep.registryUrls = [hostnameMatch.groups.hostname]; + } dep.depType = 'terraform'; dep.depName = moduleParts.join('/'); dep.depNameShort = dep.depName; From 1f7552303dd03a43ea16894e77d51067d08140fa Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Thu, 9 Jul 2020 23:33:10 +0200 Subject: [PATCH 5/8] fixup!(terraform-provider): use terraform service discovery for modules --- .../terraform-module/__snapshots__/index.spec.ts.snap | 9 ++++----- lib/datasource/terraform-module/index.spec.ts | 4 +++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap b/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap index acb9751d9570a9..c898411c25f8c2 100644 --- a/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap +++ b/lib/datasource/terraform-module/__snapshots__/index.spec.ts.snap @@ -107,7 +107,6 @@ Array [ exports[`datasource/terraform-module getReleases processes real data on changed subpath 1`] = ` Object { - "homepage": "https://registry.terraform.io/modules/hashicorp/consul/aws", "name": "hashicorp/consul/aws", "releases": Array [ Object { @@ -191,21 +190,21 @@ Array [ "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", - "host": "registry.terraform.io", + "host": "terraform.foo.bar", "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://registry.terraform.io/.well-known/terraform.json", + "url": "https://terraform.foo.bar/.well-known/terraform.json", }, Object { "headers": Object { "accept": "application/json", "accept-encoding": "gzip, deflate", - "host": "registry.terraform.io", + "host": "terraform.foo.bar", "user-agent": "https://github.com/renovatebot/renovate", }, "method": "GET", - "url": "https://registry.terraform.io/api/registry/v1/modules/hashicorp/consul/aws", + "url": "https://terraform.foo.bar/api/registry/v1/modules/hashicorp/consul/aws", }, ] `; diff --git a/lib/datasource/terraform-module/index.spec.ts b/lib/datasource/terraform-module/index.spec.ts index 544420715b18a4..a5d7a0f1331d22 100644 --- a/lib/datasource/terraform-module/index.spec.ts +++ b/lib/datasource/terraform-module/index.spec.ts @@ -14,6 +14,7 @@ const serviceDiscoveryCustomResult: any = fs.readFileSync( ); const baseUrl = 'https://registry.terraform.io'; +const localTerraformEnterprisebaseUrl = 'https://terraform.foo.bar'; describe('datasource/terraform-module', () => { describe('getReleases', () => { @@ -131,13 +132,14 @@ describe('datasource/terraform-module', () => { }); it('processes real data on changed subpath', async () => { httpMock - .scope(baseUrl) + .scope(localTerraformEnterprisebaseUrl) .get('/api/registry/v1/modules/hashicorp/consul/aws') .reply(200, consulData) .get('/.well-known/terraform.json') .reply(200, serviceDiscoveryCustomResult); const res = await getPkgReleases({ datasource, + registryUrls: ['terraform.foo.bar'], depName: 'hashicorp/consul/aws', }); expect(res).toMatchSnapshot(); From 96a7fc4cf37780a8e2d7d2a33a06f6ff67ab1027 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Fri, 10 Jul 2020 09:27:52 +0200 Subject: [PATCH 6/8] fix(terraform-provider): adding protocol to registry URL --- lib/manager/terraform/__snapshots__/extract.spec.ts.snap | 6 +++--- lib/manager/terraform/modules.ts | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap index 7ef5ccbd7c8493..6f4f5802a08fae 100644 --- a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap @@ -155,7 +155,7 @@ Object { "depNameShort": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depType": "terraform", "registryUrls": Array [ - "app.terraform.io", + "https://app.terraform.io", ], }, Object { @@ -165,7 +165,7 @@ Object { "depNameShort": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depType": "terraform", "registryUrls": Array [ - "app.terraform.io", + "https://app.terraform.io", ], }, Object { @@ -175,7 +175,7 @@ Object { "depNameShort": "app.terraform.io/example-corp/k8s-cluster/azurerm", "depType": "terraform", "registryUrls": Array [ - "app.terraform.io", + "https://app.terraform.io", ], }, Object { diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts index a60076a1a33584..87b8cd07c7e9e2 100644 --- a/lib/manager/terraform/modules.ts +++ b/lib/manager/terraform/modules.ts @@ -10,7 +10,7 @@ import { ExtractionResult, TerraformDependencyTypes } from './util'; const githubRefMatchRegex = /github.com([/:])(?[^/]+\/[a-z0-9-.]+).*\?ref=(?.*)$/; const gitTagsRefMatchRegex = /(?:git::)?(?(?:http|https|ssh):\/\/(?:.*@)?(?.*.*\/(?.*\/.*)))\?ref=(?.*)$/; -const hostnameMatchRegex = /^(http(s)?:\/\/)?(?([\w|\d]+\.)+[\w|\d]+)/; +const hostnameMatchRegex = /^(?([\w|\d]+\.)+[\w|\d]+)/; export function extractTerraformModule( startingLine: number, @@ -65,7 +65,7 @@ export function analyseTerraformModule(dep: PackageDependency): void { } else if (moduleParts.length >= 3) { const hostnameMatch = hostnameMatchRegex.exec(dep.managerData.source); if (hostnameMatch) { - dep.registryUrls = [hostnameMatch.groups.hostname]; + dep.registryUrls = [`https://${hostnameMatch.groups.hostname}`]; } dep.depType = 'terraform'; dep.depName = moduleParts.join('/'); From 639b8432ffb0b54e24f5cd8a82d19bb92016dc04 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Fri, 10 Jul 2020 09:37:17 +0200 Subject: [PATCH 7/8] Revert "chore(terraform-provider): removing datasource caching in favor of global cache" This reverts commit 93730645 --- lib/datasource/terraform-module/index.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index b459c30f31708c..ff8ccf9832da90 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -1,5 +1,6 @@ import { logger } from '../../logger'; import { ExternalHostError } from '../../types/errors/external-host-error'; +import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; @@ -68,6 +69,16 @@ export async function getReleases({ { registry, terraformRepository: repository }, 'terraform.getDependencies()' ); + const cacheNamespace = 'terraform-module'; + const cacheURL = `${registry}/${repository}`; + const cachedResult = await packageCache.get( + cacheNamespace, + cacheURL + ); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } try { const serviceDiscovery = ( await http.getJson( @@ -98,6 +109,8 @@ export async function getReleases({ dep.homepage = `https://registry.terraform.io/modules/${repository}`; } logger.trace({ dep }, 'dep'); + const cacheMinutes = 30; + await packageCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes); return dep; } catch (err) { const failureCodes = ['EAI_AGAIN']; From 614f89156a1d92384c5050535bd4d06f1a53f598 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 11 Jul 2020 14:09:19 +0200 Subject: [PATCH 8/8] implement terraform service-discovery caching --- lib/datasource/terraform-module/index.ts | 36 ++++++++++++++++++---- lib/datasource/terraform-provider/index.ts | 10 +++--- 2 files changed, 34 insertions(+), 12 deletions(-) diff --git a/lib/datasource/terraform-module/index.ts b/lib/datasource/terraform-module/index.ts index ff8ccf9832da90..d096500a603a5c 100644 --- a/lib/datasource/terraform-module/index.ts +++ b/lib/datasource/terraform-module/index.ts @@ -50,6 +50,33 @@ export interface ServiceDiscoveryResult { 'providers.v1'?: string; } +export async function getTerraformServiceDiscoveryResult( + registryUrl: string +): Promise { + const discoveryURL = `${registryUrl}/.well-known/terraform.json`; + const cacheNamespace = 'terraform-service-discovery'; + const cachedResult = await packageCache.get( + cacheNamespace, + registryUrl + ); + // istanbul ignore if + if (cachedResult) { + return cachedResult; + } + const serviceDiscovery = ( + await http.getJson(discoveryURL) + ).body; + + const cacheMinutes = 1440; // 24h + await packageCache.set( + cacheNamespace, + registryUrl, + serviceDiscovery, + cacheMinutes + ); + + return serviceDiscovery; +} /** * terraform.getReleases * @@ -80,12 +107,9 @@ export async function getReleases({ return cachedResult; } try { - const serviceDiscovery = ( - await http.getJson( - `${registryUrl}/.well-known/terraform.json` - ) - ).body; - + const serviceDiscovery = await getTerraformServiceDiscoveryResult( + registryUrl + ); const pkgUrl = `${registry}${serviceDiscovery['modules.v1']}${repository}`; const res = (await http.getJson(pkgUrl)).body; const returnedName = res.namespace + '/' + res.name + '/' + res.provider; diff --git a/lib/datasource/terraform-provider/index.ts b/lib/datasource/terraform-provider/index.ts index 5b5fba1390aea4..f97a3608482243 100644 --- a/lib/datasource/terraform-provider/index.ts +++ b/lib/datasource/terraform-provider/index.ts @@ -3,7 +3,7 @@ import { logger } from '../../logger'; import * as packageCache from '../../util/cache/package'; import { Http } from '../../util/http'; import { GetReleasesConfig, ReleaseResult } from '../common'; -import { ServiceDiscoveryResult } from '../terraform-module'; +import { getTerraformServiceDiscoveryResult } from '../terraform-module'; export const id = 'terraform-provider'; export const defaultRegistryUrls = [ @@ -38,11 +38,9 @@ async function queryRegistry( registryURL: string, repository: string ): Promise { - const serviceDiscovery = ( - await http.getJson( - `${registryURL}/.well-known/terraform.json` - ) - ).body; + const serviceDiscovery = await getTerraformServiceDiscoveryResult( + registryURL + ); const backendURL = `${registryURL}${serviceDiscovery['providers.v1']}${repository}`; const res = (await http.getJson(backendURL)).body; const dep: ReleaseResult = {