From 2427ff8b9c84c77d36eefe2bba358dc466dce4bc Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Fri, 19 Jun 2020 12:36:14 +0200 Subject: [PATCH 1/6] feat(terraform-provider): renovate required_providers --- lib/manager/terraform/__fixtures__/1.tf | 13 +++++ .../__snapshots__/extract.spec.ts.snap | 21 ++++++++ lib/manager/terraform/extract.spec.ts | 7 ++- lib/manager/terraform/extract.ts | 52 ++++++++++++++----- 4 files changed, 80 insertions(+), 13 deletions(-) diff --git a/lib/manager/terraform/__fixtures__/1.tf b/lib/manager/terraform/__fixtures__/1.tf index 793474273fa3b1..ccea08fff7ae99 100644 --- a/lib/manager/terraform/__fixtures__/1.tf +++ b/lib/manager/terraform/__fixtures__/1.tf @@ -159,3 +159,16 @@ module "gittags_http" { module "gittags_ssh" { source = "git::ssh://git@bitbucket.com/hashicorp/example?ref=v1.0.3" } + +terraform { + required_providers { + aws = ">= 2.7.0" + } +} + +terraform { + required_providers { + aws = ">= 2.5.0" + azurerm = ">= 2.0.0" + } +} diff --git a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap index 7fd0c2231ce5d8..38c6cd2323bb34 100644 --- a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap @@ -192,6 +192,27 @@ Object { "depType": "gitTags", "lookupName": "ssh://git@bitbucket.com/hashicorp/example", }, + Object { + "currentValue": ">= 2.7.0", + "datasource": "terraform-provider", + "depName": "aws", + "depNameShort": "aws", + "depType": "terraform", + }, + Object { + "currentValue": ">= 2.5.0", + "datasource": "terraform-provider", + "depName": "aws", + "depNameShort": "aws", + "depType": "terraform", + }, + Object { + "currentValue": ">= 2.0.0", + "datasource": "terraform-provider", + "depName": "azurerm", + "depNameShort": "azurerm", + "depType": "terraform", + }, ], } `; diff --git a/lib/manager/terraform/extract.spec.ts b/lib/manager/terraform/extract.spec.ts index 5e833e462e27cd..9b3e94c93504f8 100644 --- a/lib/manager/terraform/extract.spec.ts +++ b/lib/manager/terraform/extract.spec.ts @@ -19,7 +19,7 @@ describe('lib/manager/terraform/extract', () => { it('extracts', () => { const res = extractPackageFile(tf1); expect(res).toMatchSnapshot(); - expect(res.deps).toHaveLength(27); + expect(res.deps).toHaveLength(30); expect(res.deps.filter((dep) => dep.skipReason)).toHaveLength(6); }); it('returns null if only local deps', () => { @@ -42,6 +42,11 @@ describe('lib/manager/terraform/extract', () => { TerraformDependencyTypes.unknown ); }); + it('returns TerraformDependencyTypes.required_providers', () => { + expect(getTerraformDependencyType('required_providers')).toBe( + TerraformDependencyTypes.required_providers + ); + }); it('returns TerraformDependencyTypes.unknown on empty string', () => { expect(getTerraformDependencyType('')).toBe( TerraformDependencyTypes.unknown diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 1419501ef2957e..ad47c323e7a57d 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -11,6 +11,7 @@ export enum TerraformDependencyTypes { unknown = 'unknown', module = 'module', provider = 'provider', + required_providers = 'required_providers', } export function getTerraformDependencyType( @@ -23,6 +24,9 @@ export function getTerraformDependencyType( case 'provider': { return TerraformDependencyTypes.provider; } + case 'required_providers': { + return TerraformDependencyTypes.required_providers; + } default: { return TerraformDependencyTypes.unknown; } @@ -31,7 +35,11 @@ export function getTerraformDependencyType( export function extractPackageFile(content: string): PackageFile | null { logger.trace({ content }, 'terraform.extractPackageFile()'); - if (!content.includes('module "') && !content.includes('provider "')) { + if ( + !content.includes('module "') && + !content.includes('provider "') && + !content.includes('required_providers ') + ) { return null; } const deps: PackageDependency[] = []; @@ -39,25 +47,43 @@ export function extractPackageFile(content: string): PackageFile | null { const lines = content.split('\n'); for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { let line = lines[lineNumber]; - const terraformDependency = /^(module|provider)\s+"([^"]+)"\s+{\s*$/.exec( + const terraformDependency = /^\s*(module|provider|required_providers)\s+("([^"]+)"\s+)?{\s*$/.exec( line ); if (terraformDependency) { logger.trace(`Matched ${terraformDependency[1]} on line ${lineNumber}`); - const tfDepType: TerraformDependencyTypes = getTerraformDependencyType( - terraformDependency[1] - ); - const dep: PackageDependency = { - managerData: { - moduleName: terraformDependency[2], - terraformDependencyType: tfDepType, - }, - }; + const tfDepType = getTerraformDependencyType(terraformDependency[1]); + if (tfDepType === TerraformDependencyTypes.unknown) { /* istanbul ignore next */ logger.trace( `Could not identify TerraformDependencyType ${terraformDependency[1]} on line ${lineNumber}.` ); + } else if (tfDepType === TerraformDependencyTypes.required_providers) { + do { + const dep: PackageDependency = { + managerData: { + terraformDependencyType: tfDepType, + }, + }; + + lineNumber += 1; + line = lines[lineNumber]; + const kvMatch = /^\s*([^\s]+)\s+=\s+"([^"]+)"\s*$/.exec(line); + if (kvMatch) { + const [, key, value] = kvMatch; + dep.currentValue = value; + dep.managerData.moduleName = key; + dep.managerData.versionLine = lineNumber; + deps.push(dep); + } + } while (line.trim() !== '}'); } else { + const dep: PackageDependency = { + managerData: { + moduleName: terraformDependency[3], + terraformDependencyType: tfDepType, + }, + }; do { lineNumber += 1; line = lines[lineNumber]; @@ -140,7 +166,9 @@ export function extractPackageFile(content: string): PackageFile | null { } } else if ( dep.managerData.terraformDependencyType === - TerraformDependencyTypes.provider + TerraformDependencyTypes.provider || + dep.managerData.terraformDependencyType === + TerraformDependencyTypes.required_providers ) { dep.depType = 'terraform'; dep.depName = dep.managerData.moduleName; From 95b42dca1e3005b12ebd8f1c8a594090fd564fe1 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Fri, 19 Jun 2020 14:13:54 +0200 Subject: [PATCH 2/6] chore(terraform-provider): move regex into module scope --- lib/manager/terraform/extract.ts | 37 +++++++++++++++++--------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index ad47c323e7a57d..51280a4ae433c3 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -33,6 +33,9 @@ export function getTerraformDependencyType( } } +const dependencyBlockExtractionRegex = /^\s*(?module|provider|required_providers)\s+("(?[^"]+)"\s+)?{\s*$/; +const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; // extracts `exampleKey = exampleValue` + export function extractPackageFile(content: string): PackageFile | null { logger.trace({ content }, 'terraform.extractPackageFile()'); if ( @@ -47,16 +50,18 @@ export function extractPackageFile(content: string): PackageFile | null { const lines = content.split('\n'); for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { let line = lines[lineNumber]; - const terraformDependency = /^\s*(module|provider|required_providers)\s+("([^"]+)"\s+)?{\s*$/.exec( - line - ); + const terraformDependency = dependencyBlockExtractionRegex.exec(line); if (terraformDependency) { - logger.trace(`Matched ${terraformDependency[1]} on line ${lineNumber}`); - const tfDepType = getTerraformDependencyType(terraformDependency[1]); + logger.trace( + `Matched ${terraformDependency.groups.type} on line ${lineNumber}` + ); + const tfDepType = getTerraformDependencyType( + terraformDependency.groups.type + ); if (tfDepType === TerraformDependencyTypes.unknown) { /* istanbul ignore next */ logger.trace( - `Could not identify TerraformDependencyType ${terraformDependency[1]} on line ${lineNumber}.` + `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.` ); } else if (tfDepType === TerraformDependencyTypes.required_providers) { do { @@ -68,11 +73,10 @@ export function extractPackageFile(content: string): PackageFile | null { lineNumber += 1; line = lines[lineNumber]; - const kvMatch = /^\s*([^\s]+)\s+=\s+"([^"]+)"\s*$/.exec(line); + const kvMatch = keyValueExtractionRegex.exec(line); if (kvMatch) { - const [, key, value] = kvMatch; - dep.currentValue = value; - dep.managerData.moduleName = key; + dep.currentValue = kvMatch.groups.value; + dep.managerData.moduleName = kvMatch.groups.key; dep.managerData.versionLine = lineNumber; deps.push(dep); } @@ -80,21 +84,20 @@ export function extractPackageFile(content: string): PackageFile | null { } else { const dep: PackageDependency = { managerData: { - moduleName: terraformDependency[3], + moduleName: terraformDependency.groups.lookupName, terraformDependencyType: tfDepType, }, }; do { lineNumber += 1; line = lines[lineNumber]; - const kvMatch = /^\s*([^\s]+)\s+=\s+"([^"]+)"\s*$/.exec(line); + const kvMatch = keyValueExtractionRegex.exec(line); if (kvMatch) { - const [, key, value] = kvMatch; - if (key === 'version') { - dep.currentValue = value; + if (kvMatch.groups.key === 'version') { + dep.currentValue = kvMatch.groups.value; dep.managerData.versionLine = lineNumber; - } else if (key === 'source') { - dep.managerData.source = value; + } else if (kvMatch.groups.key === 'source') { + dep.managerData.source = kvMatch.groups.value; dep.managerData.sourceLine = lineNumber; } } From 9760672787f8719458e4cc53dd3fe9f4f67f830e Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 20 Jun 2020 11:34:59 +0200 Subject: [PATCH 3/6] chore(terraform-provider): remove deprecated line number field --- lib/manager/terraform/extract.ts | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 51280a4ae433c3..9f83e69c0db0ec 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -129,7 +129,6 @@ export function extractPackageFile(content: string): PackageFile | null { dep.currentValue = githubRefMatch[3]; dep.datasource = datasourceGithubTags.id; dep.lookupName = depNameShort; - dep.managerData.lineNumber = dep.managerData.sourceLine; if (!isVersion(dep.currentValue)) { dep.skipReason = SkipReason.UnsupportedVersion; } @@ -148,7 +147,6 @@ export function extractPackageFile(content: string): PackageFile | null { } dep.currentValue = gitTagsRefMatch[4]; dep.datasource = datasourceGitTags.id; - dep.managerData.lineNumber = dep.managerData.sourceLine; if (!isVersion(dep.currentValue)) { dep.skipReason = SkipReason.UnsupportedVersion; } @@ -160,7 +158,6 @@ export function extractPackageFile(content: string): PackageFile | null { dep.depType = 'terraform'; dep.depName = moduleParts.join('/'); dep.depNameShort = dep.depName; - dep.managerData.lineNumber = dep.managerData.versionLine; dep.datasource = datasourceTerraformModule.id; } } else { @@ -176,9 +173,8 @@ export function extractPackageFile(content: string): PackageFile | null { dep.depType = 'terraform'; dep.depName = dep.managerData.moduleName; dep.depNameShort = dep.managerData.moduleName; - dep.managerData.lineNumber = dep.managerData.versionLine; dep.datasource = datasourceTerraformProvider.id; - if (dep.managerData.lineNumber) { + if (dep.managerData.versionLine) { if (!isValid(dep.currentValue)) { dep.skipReason = SkipReason.UnsupportedVersion; } From 5d5d98430f2b1dcfc9951873feac6d8a170a82eb Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 20 Jun 2020 13:34:41 +0200 Subject: [PATCH 4/6] chore(contributors): add sebastian.poxhofer to contributors --- package.json | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/package.json b/package.json index 5c90cec5be10bd..4f07370b85e910 100644 --- a/package.json +++ b/package.json @@ -95,7 +95,8 @@ "Tanuel ", "Viral Ruparel ", "Vladimir Starkov ", - "Mikhail Yakushin " + "Mikhail Yakushin ", + "Sebastian Poxhofer " ], "license": "AGPL-3.0", "bugs": { From 5a80bada9553680098192e26b374c6c31503a34b Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 20 Jun 2020 13:36:57 +0200 Subject: [PATCH 5/6] chore(terraform-provider): increase log level for unknown dependency type --- lib/manager/terraform/extract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 9f83e69c0db0ec..ca102193ba8d9d 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -60,7 +60,7 @@ export function extractPackageFile(content: string): PackageFile | null { ); if (tfDepType === TerraformDependencyTypes.unknown) { - /* istanbul ignore next */ logger.trace( + /* istanbul ignore next */ logger.warn( `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.` ); } else if (tfDepType === TerraformDependencyTypes.required_providers) { From ea978c620b7412508b1101b0aca30d19ca5686af Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sat, 27 Jun 2020 11:57:04 +0200 Subject: [PATCH 6/6] chore(terraform-provider): remove unnecessary skip.reason --- lib/manager/terraform/__snapshots__/extract.spec.ts.snap | 2 +- lib/manager/terraform/extract.ts | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap index 38c6cd2323bb34..195e5888bf9b42 100644 --- a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap @@ -140,7 +140,7 @@ Object { "depName": "helm", "depNameShort": "helm", "depType": "terraform", - "skipReason": "no-version", + "skipReason": "unsupported-version", }, Object { "currentValue": "V1.9", diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index ca102193ba8d9d..dad95431b6ee61 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -174,12 +174,8 @@ export function extractPackageFile(content: string): PackageFile | null { dep.depName = dep.managerData.moduleName; dep.depNameShort = dep.managerData.moduleName; dep.datasource = datasourceTerraformProvider.id; - if (dep.managerData.versionLine) { - if (!isValid(dep.currentValue)) { - dep.skipReason = SkipReason.UnsupportedVersion; - } - } else if (!dep.skipReason) { - dep.skipReason = SkipReason.NoVersion; + if (!isValid(dep.currentValue)) { + dep.skipReason = SkipReason.UnsupportedVersion; } } delete dep.managerData;