Skip to content

Commit

Permalink
feat(terraform-provider): implement secondary release source
Browse files Browse the repository at this point in the history
  • Loading branch information
@secustor
secustor committed Jun 18, 2020
1 parent bd98334 commit 961fdaf
Show file tree
Hide file tree
Showing 4 changed files with 258 additions and 29 deletions.
103 changes: 103 additions & 0 deletions lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json
View file
@@ -0,0 +1,103 @@
{
"terraform-provider-google-beta": {
"name": "terraform-provider-google-beta",
"versions": {
"1.19.0": {
"name": "terraform-provider-google-beta",
"version": "1.19.0",
"shasums": "terraform-provider-google-beta_1.19.0_SHA256SUMS",
"shasums_signature": "terraform-provider-google-beta_1.19.0_SHA256SUMS.sig",
"builds": [
{
"name": "terraform-provider-google-beta",
"version": "1.19.0",
"os": "darwin",
"arch": "amd64",
"filename": "terraform-provider-google-beta_1.19.0_darwin_amd64.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_darwin_amd64.zip"
},
{
"name": "terraform-provider-google-beta",
"version": "1.19.0",
"os": "freebsd",
"arch": "386",
"filename": "terraform-provider-google-beta_1.19.0_freebsd_386.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_386.zip"
},
{
"name": "terraform-provider-google-beta",
"version": "1.19.0",
"os": "freebsd",
"arch": "amd64",
"filename": "terraform-provider-google-beta_1.19.0_freebsd_amd64.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.19.0/terraform-provider-google-beta_1.19.0_freebsd_amd64.zip"
}
]
},
"1.20.0": {
"name": "terraform-provider-google-beta",
"version": "1.20.0",
"shasums": "terraform-provider-google-beta_1.20.0_SHA256SUMS",
"shasums_signature": "terraform-provider-google-beta_1.20.0_SHA256SUMS.sig",
"builds": [
{
"name": "terraform-provider-google-beta",
"version": "1.20.0",
"os": "openbsd",
"arch": "386",
"filename": "terraform-provider-google-beta_1.20.0_openbsd_386.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_386.zip"
},
{
"name": "terraform-provider-google-beta",
"version": "1.20.0",
"os": "openbsd",
"arch": "amd64",
"filename": "terraform-provider-google-beta_1.20.0_openbsd_amd64.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_openbsd_amd64.zip"
},
{
"name": "terraform-provider-google-beta",
"version": "1.20.0",
"os": "solaris",
"arch": "amd64",
"filename": "terraform-provider-google-beta_1.20.0_solaris_amd64.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/1.20.0/terraform-provider-google-beta_1.20.0_solaris_amd64.zip"
}
]
},
"2.0.0": {
"name": "terraform-provider-google-beta",
"version": "2.0.0",
"shasums": "terraform-provider-google-beta_2.0.0_SHA256SUMS",
"shasums_signature": "terraform-provider-google-beta_2.0.0_SHA256SUMS.sig",
"builds": [
{
"name": "terraform-provider-google-beta",
"version": "2.0.0",
"os": "darwin",
"arch": "amd64",
"filename": "terraform-provider-google-beta_2.0.0_darwin_amd64.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_darwin_amd64.zip"
},
{
"name": "terraform-provider-google-beta",
"version": "2.0.0",
"os": "freebsd",
"arch": "386",
"filename": "terraform-provider-google-beta_2.0.0_freebsd_386.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_386.zip"
},
{
"name": "terraform-provider-google-beta",
"version": "2.0.0",
"os": "freebsd",
"arch": "amd64",
"filename": "terraform-provider-google-beta_2.0.0_freebsd_amd64.zip",
"url": "https://releases.hashicorp.com/terraform-provider-google-beta/2.0.0/terraform-provider-google-beta_2.0.0_freebsd_amd64.zip"
}
]
}
}
}
}
43 changes: 43 additions & 0 deletions lib/datasource/terraform-provider/__snapshots__/index.spec.ts.snap
View file
@@ -1,5 +1,48 @@
// Jest Snapshot v1, https://goo.gl/fbAQLP

exports[`datasource/terraform getReleases processes data with alternative backend 1`] = `
Object {
"name": "hashicorp/google-beta",
"releases": Array [
Object {
"version": "1.19.0",
},
Object {
"version": "1.20.0",
},
Object {
"version": "2.0.0",
},
],
"versions": Object {},
}
`;

exports[`datasource/terraform getReleases processes data with alternative backend 2`] = `
Array [
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "registry.terraform.io",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://registry.terraform.io/v1/providers/hashicorp/google-beta",
},
Object {
"headers": Object {
"accept": "application/json",
"accept-encoding": "gzip, deflate",
"host": "releases.hashicorp.com",
"user-agent": "https://github.com/renovatebot/renovate",
},
"method": "GET",
"url": "https://releases.hashicorp.com/index.json",
},
]
`;

exports[`datasource/terraform getReleases processes real data 1`] = `
Object {
"homepage": "https://registry.terraform.io/providers/hashicorp/azurerm",
Expand Down
22 changes: 22 additions & 0 deletions lib/datasource/terraform-provider/index.spec.ts
View file
Expand Up @@ -6,6 +6,9 @@ import { id as datasource } from '.';
const consulData: any = fs.readFileSync(
'lib/datasource/terraform-provider/__fixtures__/azurerm-provider.json'
);
const hashicorpReleases: any = fs.readFileSync(
'lib/datasource/terraform-provider/__fixtures__/releaseBackendIndex.json'
);

const baseUrl = 'https://registry.terraform.io/';

Expand Down Expand Up @@ -69,5 +72,24 @@ describe('datasource/terraform', () => {
expect(res).not.toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
it('processes data with alternative backend', async () => {
httpMock
.scope('https://registry.terraform.io')
.get('/v1/providers/hashicorp/google-beta')
.reply(404, {
errors: ['Not Found'],
});
httpMock
.scope('https://releases.hashicorp.com')
.get('/index.json')
.reply(200, JSON.parse(hashicorpReleases));

const res = await terraformProvider.getReleases({
lookupName: 'google-beta',
});
expect(res).toMatchSnapshot();
expect(res).not.toBeNull();
expect(httpMock.getTrace()).toMatchSnapshot();
});
});
});
119 changes: 90 additions & 29 deletions lib/datasource/terraform-provider/index.ts
View file
Expand Up @@ -15,6 +15,84 @@ interface TerraformProvider {
versions: string[];
}

interface TerraformProviderReleaseBackend {
[key: string]: {
name: string;
versions: VersionsReleaseBackend;
};
}

interface VersionsReleaseBackend {
[key: string]: Record<string, any>;
}

async function queryRegistry(
lookupName: string,
backendURL: string,
repository: string
): Promise<ReleaseResult> {
try {
const res = (await http.getJson<TerraformProvider>(backendURL)).body;
const dep: ReleaseResult = {
name: repository,
versions: {},
releases: null,
};
if (res.source) {
dep.sourceUrl = res.source;
}
dep.releases = res.versions.map((version) => ({
version,
}));
dep.homepage = `https://registry.terraform.io/providers/${repository}`;
logger.trace({ dep }, 'dep');
return dep;
} catch (err) {
logger.debug(
{ lookupName },
`Terraform registry ("registry.terraform.io") lookup failure: not found`
);
logger.debug({
err,
});
return null;
}
}

async function queryReleaseBackend(
lookupName: string,
backendURL: string,
repository: string
): Promise<ReleaseResult> {
const backendLookUpName = `terraform-provider-${lookupName}`;
try {
const res = (
await http.getJson<TerraformProviderReleaseBackend>(backendURL)
).body;
const dep: ReleaseResult = {
name: repository,
versions: {},
releases: null,
};
dep.releases = Object.keys(res[backendLookUpName].versions).map(
(version) => ({
version,
})
);
logger.trace({ dep }, 'dep');
return dep;
} catch (err) {
logger.debug(
{ lookupName },
`Terraform registry ("releases.hashicorp.com") lookup failure: not found`
);
logger.debug({
err,
});
return null;
}
}

/**
* terraform-provider.getReleases
*
Expand All @@ -26,49 +104,32 @@ export async function getReleases({
}: GetReleasesConfig): Promise<ReleaseResult | null> {
const repository = `hashicorp/${lookupName}`;

const releasesBackendURL = `https://releases.hashicorp.com/index.json`;
const registryBackendURL = `https://registry.terraform.io/v1/providers/${repository}`;

logger.debug({ lookupName }, 'terraform-provider.getDependencies()');
const cacheNamespace = 'terraform-providers';
const pkgUrl = `https://registry.terraform.io/v1/providers/${repository}`;
const cacheMinutes = 30;
const cachedResult = await globalCache.get<ReleaseResult>(
cacheNamespace,
pkgUrl
lookupName
);
// istanbul ignore if
if (cachedResult) {
return cachedResult;
}
try {
const res = (await http.getJson<TerraformProvider>(pkgUrl)).body;
// Simplify response before caching and returning
const dep: ReleaseResult = {
name: repository,
versions: {},
releases: null,
};
if (res.source) {
dep.sourceUrl = res.source;
let dep = await queryRegistry(lookupName, registryBackendURL, repository);
if (dep) {
await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes);
return dep;
}
dep.releases = res.versions.map((version) => ({
version,
}));
if (pkgUrl.startsWith('https://registry.terraform.io/')) {
dep.homepage = `https://registry.terraform.io/providers/${repository}`;
dep = await queryReleaseBackend(lookupName, releasesBackendURL, repository);
if (dep) {
await globalCache.set(cacheNamespace, lookupName, dep, cacheMinutes);
}
logger.trace({ dep }, 'dep');
const cacheMinutes = 30;
await globalCache.set(cacheNamespace, pkgUrl, dep, cacheMinutes);
return dep;
} catch (err) {
if (err.statusCode === 404 || err.code === 'ENOTFOUND') {
logger.debug(
{ lookupName },
`Terraform registry lookup failure: not found`
);
logger.debug({
err,
});
return null;
}
const failureCodes = ['EAI_AGAIN'];
// istanbul ignore if
if (failureCodes.includes(err.code)) {
Expand Down

0 comments on commit 961fdaf

Please sign in to comment.