From 0af36ae982da95889ae777ea33912b61b585549d Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sun, 28 Jun 2020 20:38:27 +0200 Subject: [PATCH 1/9] chore(terraform-provider): replace if chain with array.some() --- lib/manager/terraform/extract.ts | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index dad95431b6ee61..78c81e8be764b3 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -33,16 +33,22 @@ export function getTerraformDependencyType( } } +function checkFileContainsDependency( + content: string, + checkList: string[] +): boolean { + return checkList.some((check) => { + return content.includes(check); + }); +} + const dependencyBlockExtractionRegex = /^\s*(?module|provider|required_providers)\s+("(?[^"]+)"\s+)?{\s*$/; +const contentCheckList = ['module "', 'provider "', 'required_providers ']; const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; // extracts `exampleKey = exampleValue` export function extractPackageFile(content: string): PackageFile | null { logger.trace({ content }, 'terraform.extractPackageFile()'); - if ( - !content.includes('module "') && - !content.includes('provider "') && - !content.includes('required_providers ') - ) { + if (!checkFileContainsDependency(content, contentCheckList)) { return null; } const deps: PackageDependency[] = []; From 914ac1da6140235510ca342bb391190e9a3fc256 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Sun, 28 Jun 2020 22:29:31 +0200 Subject: [PATCH 2/9] fix(terraform-provider): make git:: prefix optional --- .../terraform/__snapshots__/extract.spec.ts.snap | 9 ++++++++- lib/manager/terraform/extract.ts | 10 ++++------ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap index 195e5888bf9b42..e61646a6faa8bd 100644 --- a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap @@ -150,7 +150,14 @@ Object { "depType": "terraform", "skipReason": "unsupported-version", }, - Object {}, + Object { + "currentValue": "v1.0.0", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "https://bitbucket.com/hashicorp/example", + }, Object { "currentValue": "v1.0.0", "datasource": "git-tags", diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 78c81e8be764b3..3518ebc2f95bc3 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -45,6 +45,8 @@ function checkFileContainsDependency( const dependencyBlockExtractionRegex = /^\s*(?module|provider|required_providers)\s+("(?[^"]+)"\s+)?{\s*$/; const contentCheckList = ['module "', 'provider "', 'required_providers ']; const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; // extracts `exampleKey = exampleValue` +const githubRefMatchRegex = /github.com(\/|:)([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/; +const gitTagsRefMatchRegex = /(?:git::)?((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/; export function extractPackageFile(content: string): PackageFile | null { logger.trace({ content }, 'terraform.extractPackageFile()'); @@ -120,12 +122,8 @@ export function extractPackageFile(content: string): PackageFile | null { dep.managerData.terraformDependencyType === TerraformDependencyTypes.module ) { - const githubRefMatch = /github.com(\/|:)([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/.exec( - dep.managerData.source - ); - const gitTagsRefMatch = /git::((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/.exec( - dep.managerData.source - ); + const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source); + const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source); /* eslint-disable no-param-reassign */ if (githubRefMatch) { const depNameShort = githubRefMatch[2].replace(/\.git$/, ''); From 116af85a7dc58d117e84bef02b785925c0ab3d11 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Mon, 29 Jun 2020 19:01:49 +0200 Subject: [PATCH 3/9] chore(terraform-provider): refactor terraform manager --- lib/manager/terraform/extract.spec.ts | 38 +--- lib/manager/terraform/extract.ts | 208 +++++--------------- lib/manager/terraform/modules.ts | 76 +++++++ lib/manager/terraform/provider.ts | 52 +++++ lib/manager/terraform/required_providers.ts | 30 +++ lib/manager/terraform/util.spec.ts | 36 ++++ lib/manager/terraform/util.ts | 45 +++++ 7 files changed, 293 insertions(+), 192 deletions(-) create mode 100644 lib/manager/terraform/modules.ts create mode 100644 lib/manager/terraform/provider.ts create mode 100644 lib/manager/terraform/required_providers.ts create mode 100644 lib/manager/terraform/util.spec.ts create mode 100644 lib/manager/terraform/util.ts diff --git a/lib/manager/terraform/extract.spec.ts b/lib/manager/terraform/extract.spec.ts index 9b3e94c93504f8..5e3fba602b7475 100644 --- a/lib/manager/terraform/extract.spec.ts +++ b/lib/manager/terraform/extract.spec.ts @@ -1,9 +1,5 @@ import { readFileSync } from 'fs'; -import { - TerraformDependencyTypes, - extractPackageFile, - getTerraformDependencyType, -} from './extract'; +import { extractPackageFile } from './extract'; const tf1 = readFileSync('lib/manager/terraform/__fixtures__/1.tf', 'utf8'); const tf2 = `module "relative" { @@ -26,36 +22,4 @@ describe('lib/manager/terraform/extract', () => { expect(extractPackageFile(tf2)).toBeNull(); }); }); - describe('getTerraformDependencyType()', () => { - it('returns TerraformDependencyTypes.module', () => { - expect(getTerraformDependencyType('module')).toBe( - TerraformDependencyTypes.module - ); - }); - it('returns TerraformDependencyTypes.provider', () => { - expect(getTerraformDependencyType('provider')).toBe( - TerraformDependencyTypes.provider - ); - }); - it('returns TerraformDependencyTypes.unknown', () => { - expect(getTerraformDependencyType('unknown')).toBe( - TerraformDependencyTypes.unknown - ); - }); - it('returns TerraformDependencyTypes.required_providers', () => { - expect(getTerraformDependencyType('required_providers')).toBe( - TerraformDependencyTypes.required_providers - ); - }); - it('returns TerraformDependencyTypes.unknown on empty string', () => { - expect(getTerraformDependencyType('')).toBe( - TerraformDependencyTypes.unknown - ); - }); - it('returns TerraformDependencyTypes.unknown on string with random chars', () => { - expect(getTerraformDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe( - TerraformDependencyTypes.unknown - ); - }); - }); }); diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 3518ebc2f95bc3..8fdc8e087657ae 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -1,63 +1,27 @@ -import * as datasourceGitTags from '../../datasource/git-tags'; -import * as datasourceGithubTags from '../../datasource/github-tags'; -import * as datasourceTerraformModule from '../../datasource/terraform-module'; -import * as datasourceTerraformProvider from '../../datasource/terraform-provider'; import { logger } from '../../logger'; -import { SkipReason } from '../../types'; -import { isValid, isVersion } from '../../versioning/hashicorp'; import { PackageDependency, PackageFile } from '../common'; - -export enum TerraformDependencyTypes { - unknown = 'unknown', - module = 'module', - provider = 'provider', - required_providers = 'required_providers', -} - -export function getTerraformDependencyType( - value: string -): TerraformDependencyTypes { - switch (value) { - case 'module': { - return TerraformDependencyTypes.module; - } - case 'provider': { - return TerraformDependencyTypes.provider; - } - case 'required_providers': { - return TerraformDependencyTypes.required_providers; - } - default: { - return TerraformDependencyTypes.unknown; - } - } -} - -function checkFileContainsDependency( - content: string, - checkList: string[] -): boolean { - return checkList.some((check) => { - return content.includes(check); - }); -} +import { analyseTerraformModule, extractTerraformModule } from './modules'; +import { analyzeTerraformProvider, extractTerraformProvider } from './provider'; +import { extractTerraformRequiredProviders } from './required_providers'; +import { + TerraformDependencyTypes, + checkFileContainsDependency, + getTerraformDependencyType, +} from './util'; const dependencyBlockExtractionRegex = /^\s*(?module|provider|required_providers)\s+("(?[^"]+)"\s+)?{\s*$/; const contentCheckList = ['module "', 'provider "', 'required_providers ']; -const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; // extracts `exampleKey = exampleValue` -const githubRefMatchRegex = /github.com(\/|:)([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/; -const gitTagsRefMatchRegex = /(?:git::)?((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/; export function extractPackageFile(content: string): PackageFile | null { logger.trace({ content }, 'terraform.extractPackageFile()'); if (!checkFileContainsDependency(content, contentCheckList)) { return null; } - const deps: PackageDependency[] = []; + let deps: PackageDependency[] = []; try { const lines = content.split('\n'); for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { - let line = lines[lineNumber]; + const line = lines[lineNumber]; const terraformDependency = dependencyBlockExtractionRegex.exec(line); if (terraformDependency) { logger.trace( @@ -66,51 +30,39 @@ export function extractPackageFile(content: string): PackageFile | null { const tfDepType = getTerraformDependencyType( terraformDependency.groups.type ); - - if (tfDepType === TerraformDependencyTypes.unknown) { - /* istanbul ignore next */ logger.warn( - `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.` - ); - } else if (tfDepType === TerraformDependencyTypes.required_providers) { - do { - const dep: PackageDependency = { - managerData: { - terraformDependencyType: tfDepType, - }, - }; - - lineNumber += 1; - line = lines[lineNumber]; - const kvMatch = keyValueExtractionRegex.exec(line); - if (kvMatch) { - dep.currentValue = kvMatch.groups.value; - dep.managerData.moduleName = kvMatch.groups.key; - dep.managerData.versionLine = lineNumber; - deps.push(dep); - } - } while (line.trim() !== '}'); - } else { - const dep: PackageDependency = { - managerData: { - moduleName: terraformDependency.groups.lookupName, - terraformDependencyType: tfDepType, - }, - }; - do { - lineNumber += 1; - line = lines[lineNumber]; - const kvMatch = keyValueExtractionRegex.exec(line); - if (kvMatch) { - if (kvMatch.groups.key === 'version') { - dep.currentValue = kvMatch.groups.value; - dep.managerData.versionLine = lineNumber; - } else if (kvMatch.groups.key === 'source') { - dep.managerData.source = kvMatch.groups.value; - dep.managerData.sourceLine = lineNumber; - } - } - } while (line.trim() !== '}'); - deps.push(dep); + let result = null; + switch (tfDepType) { + case TerraformDependencyTypes.required_providers: { + result = extractTerraformRequiredProviders(lineNumber, lines); + break; + } + case TerraformDependencyTypes.provider: { + result = extractTerraformProvider( + lineNumber, + lines, + terraformDependency.groups.lookupName + ); + break; + } + case TerraformDependencyTypes.module: { + result = extractTerraformModule( + lineNumber, + lines, + terraformDependency.groups.lookupName + ); + break; + } + case TerraformDependencyTypes.unknown: + default: + logger.warn( + `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.` + ); + break; + } + if (result) { + lineNumber = result.lineNumber; + deps = deps.concat(result.dependencies); + result = null; } } } @@ -118,72 +70,18 @@ export function extractPackageFile(content: string): PackageFile | null { logger.warn({ err }, 'Error extracting buildkite plugins'); } deps.forEach((dep) => { - if ( - dep.managerData.terraformDependencyType === - TerraformDependencyTypes.module - ) { - const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source); - const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source); - /* eslint-disable no-param-reassign */ - if (githubRefMatch) { - const depNameShort = githubRefMatch[2].replace(/\.git$/, ''); - dep.depType = 'github'; - dep.depName = 'github.com/' + depNameShort; - dep.depNameShort = depNameShort; - dep.currentValue = githubRefMatch[3]; - dep.datasource = datasourceGithubTags.id; - dep.lookupName = depNameShort; - if (!isVersion(dep.currentValue)) { - dep.skipReason = SkipReason.UnsupportedVersion; - } - } else if (gitTagsRefMatch) { - dep.depType = 'gitTags'; - if (gitTagsRefMatch[2].includes('//')) { - logger.debug('Terraform module contains subdirectory'); - dep.depName = gitTagsRefMatch[2].split('//')[0]; - dep.depNameShort = dep.depName.split(/\/(.+)/)[1]; - const tempLookupName = gitTagsRefMatch[1].split('//'); - dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1]; - } else { - dep.depName = gitTagsRefMatch[2].replace('.git', ''); - dep.depNameShort = gitTagsRefMatch[3].replace('.git', ''); - dep.lookupName = gitTagsRefMatch[1]; - } - dep.currentValue = gitTagsRefMatch[4]; - dep.datasource = datasourceGitTags.id; - if (!isVersion(dep.currentValue)) { - dep.skipReason = SkipReason.UnsupportedVersion; - } - } else if (dep.managerData.source) { - const moduleParts = dep.managerData.source.split('//')[0].split('/'); - if (moduleParts[0] === '..') { - dep.skipReason = SkipReason.Local; - } else if (moduleParts.length >= 3) { - dep.depType = 'terraform'; - dep.depName = moduleParts.join('/'); - dep.depNameShort = dep.depName; - dep.datasource = datasourceTerraformModule.id; - } - } else { - logger.debug({ dep }, 'terraform dep has no source'); - dep.skipReason = SkipReason.NoSource; - } - } else if ( - dep.managerData.terraformDependencyType === - TerraformDependencyTypes.provider || - dep.managerData.terraformDependencyType === - TerraformDependencyTypes.required_providers - ) { - dep.depType = 'terraform'; - dep.depName = dep.managerData.moduleName; - dep.depNameShort = dep.managerData.moduleName; - dep.datasource = datasourceTerraformProvider.id; - if (!isValid(dep.currentValue)) { - dep.skipReason = SkipReason.UnsupportedVersion; - } + switch (dep.managerData.terraformDependencyType) { + case TerraformDependencyTypes.required_providers: + case TerraformDependencyTypes.provider: + analyzeTerraformProvider(dep); + break; + case TerraformDependencyTypes.module: + analyseTerraformModule(dep); + break; + default: } + // eslint-disable-next-line no-param-reassign delete dep.managerData; - /* eslint-enable no-param-reassign */ }); if (deps.some((dep) => dep.skipReason !== 'local')) { return { deps }; diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts new file mode 100644 index 00000000000000..a74d6ce9dedc26 --- /dev/null +++ b/lib/manager/terraform/modules.ts @@ -0,0 +1,76 @@ +import * as datasourceGitTags from '../../datasource/git-tags'; +import * as datasourceGithubTags from '../../datasource/github-tags'; +import * as datasourceTerraformModule from '../../datasource/terraform-module'; +import { logger } from '../../logger'; +import { SkipReason } from '../../types'; +import { isVersion } from '../../versioning/hashicorp'; +import { PackageDependency } from '../common'; +import { extractTerraformProvider } from './provider'; +import { + ExtractionResult, + TerraformDependencyTypes, + gitTagsRefMatchRegex, + githubRefMatchRegex, +} from './util'; + +export function extractTerraformModule( + startingLine: number, + lines: string[], + moduleName: string +): ExtractionResult { + const result = extractTerraformProvider(startingLine, lines, moduleName); + result.dependencies.forEach((dep) => { + // eslint-disable-next-line no-param-reassign + dep.managerData.terraformDependencyType = TerraformDependencyTypes.module; + }); + return result; +} + +export function analyseTerraformModule(dep: PackageDependency): void { + const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source); + const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source); + /* eslint-disable no-param-reassign */ + if (githubRefMatch) { + const depNameShort = githubRefMatch[2].replace(/\.git$/, ''); + dep.depType = 'github'; + dep.depName = 'github.com/' + depNameShort; + dep.depNameShort = depNameShort; + dep.currentValue = githubRefMatch[3]; + dep.datasource = datasourceGithubTags.id; + dep.lookupName = depNameShort; + if (!isVersion(dep.currentValue)) { + dep.skipReason = SkipReason.UnsupportedVersion; + } + } else if (gitTagsRefMatch) { + dep.depType = 'gitTags'; + if (gitTagsRefMatch[2].includes('//')) { + logger.debug('Terraform module contains subdirectory'); + dep.depName = gitTagsRefMatch[2].split('//')[0]; + dep.depNameShort = dep.depName.split(/\/(.+)/)[1]; + const tempLookupName = gitTagsRefMatch[1].split('//'); + dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1]; + } else { + dep.depName = gitTagsRefMatch[2].replace('.git', ''); + dep.depNameShort = gitTagsRefMatch[3].replace('.git', ''); + dep.lookupName = gitTagsRefMatch[1]; + } + dep.currentValue = gitTagsRefMatch[4]; + dep.datasource = datasourceGitTags.id; + if (!isVersion(dep.currentValue)) { + dep.skipReason = SkipReason.UnsupportedVersion; + } + } else if (dep.managerData.source) { + const moduleParts = dep.managerData.source.split('//')[0].split('/'); + if (moduleParts[0] === '..') { + dep.skipReason = SkipReason.Local; + } else if (moduleParts.length >= 3) { + dep.depType = 'terraform'; + dep.depName = moduleParts.join('/'); + dep.depNameShort = dep.depName; + dep.datasource = datasourceTerraformModule.id; + } + } else { + logger.debug({ dep }, 'terraform dep has no source'); + dep.skipReason = SkipReason.NoSource; + } +} diff --git a/lib/manager/terraform/provider.ts b/lib/manager/terraform/provider.ts new file mode 100644 index 00000000000000..25a4855a78a45b --- /dev/null +++ b/lib/manager/terraform/provider.ts @@ -0,0 +1,52 @@ +import * as datasourceTerraformProvider from '../../datasource/terraform-provider'; +import { SkipReason } from '../../types'; +import { isValid } from '../../versioning/hashicorp'; +import { PackageDependency } from '../common'; +import { + ExtractionResult, + TerraformDependencyTypes, + keyValueExtractionRegex, +} from './util'; + +export function extractTerraformProvider( + startingLine: number, + lines: string[], + moduleName: string +): ExtractionResult { + let lineNumber = startingLine; + let line: string; + const deps: PackageDependency[] = []; + const dep: PackageDependency = { + managerData: { + moduleName, + terraformDependencyType: TerraformDependencyTypes.provider, + }, + }; + do { + lineNumber += 1; + line = lines[lineNumber]; + const kvMatch = keyValueExtractionRegex.exec(line); + if (kvMatch) { + if (kvMatch.groups.key === 'version') { + dep.currentValue = kvMatch.groups.value; + dep.managerData.versionLine = lineNumber; + } else if (kvMatch.groups.key === 'source') { + dep.managerData.source = kvMatch.groups.value; + dep.managerData.sourceLine = lineNumber; + } + } + } while (line.trim() !== '}'); + deps.push(dep); + return { lineNumber, dependencies: deps }; +} + +export function analyzeTerraformProvider(dep: PackageDependency): void { + /* eslint-disable no-param-reassign */ + dep.depType = 'terraform'; + dep.depName = dep.managerData.moduleName; + dep.depNameShort = dep.managerData.moduleName; + dep.datasource = datasourceTerraformProvider.id; + if (!isValid(dep.currentValue)) { + dep.skipReason = SkipReason.UnsupportedVersion; + } +} diff --git a/lib/manager/terraform/required_providers.ts b/lib/manager/terraform/required_providers.ts new file mode 100644 index 00000000000000..55f9ca9ad63ced --- /dev/null +++ b/lib/manager/terraform/required_providers.ts @@ -0,0 +1,30 @@ +import { TerraformDependencyTypes } from '../../../dist/manager/terraform/extract'; +import { PackageDependency } from '../common'; +import { ExtractionResult, keyValueExtractionRegex } from './util'; + +export function extractTerraformRequiredProviders( + startingLine: number, + lines: string[] +): ExtractionResult { + let lineNumber = startingLine; + let line: string; + const deps: PackageDependency[] = []; + do { + const dep: PackageDependency = { + managerData: { + terraformDependencyType: TerraformDependencyTypes.required_providers, + }, + }; + + lineNumber += 1; + line = lines[lineNumber]; + const kvMatch = keyValueExtractionRegex.exec(line); + if (kvMatch) { + dep.currentValue = kvMatch.groups.value; + dep.managerData.moduleName = kvMatch.groups.key; + dep.managerData.versionLine = lineNumber; + deps.push(dep); + } + } while (line.trim() !== '}'); + return { lineNumber, dependencies: deps }; +} diff --git a/lib/manager/terraform/util.spec.ts b/lib/manager/terraform/util.spec.ts new file mode 100644 index 00000000000000..cb1de0af4c1379 --- /dev/null +++ b/lib/manager/terraform/util.spec.ts @@ -0,0 +1,36 @@ +import { TerraformDependencyTypes, getTerraformDependencyType } from './util'; + +describe('lib/manager/terraform/extract', () => { + describe('getTerraformDependencyType()', () => { + it('returns TerraformDependencyTypes.module', () => { + expect(getTerraformDependencyType('module')).toBe( + TerraformDependencyTypes.module + ); + }); + it('returns TerraformDependencyTypes.provider', () => { + expect(getTerraformDependencyType('provider')).toBe( + TerraformDependencyTypes.provider + ); + }); + it('returns TerraformDependencyTypes.unknown', () => { + expect(getTerraformDependencyType('unknown')).toBe( + TerraformDependencyTypes.unknown + ); + }); + it('returns TerraformDependencyTypes.required_providers', () => { + expect(getTerraformDependencyType('required_providers')).toBe( + TerraformDependencyTypes.required_providers + ); + }); + it('returns TerraformDependencyTypes.unknown on empty string', () => { + expect(getTerraformDependencyType('')).toBe( + TerraformDependencyTypes.unknown + ); + }); + it('returns TerraformDependencyTypes.unknown on string with random chars', () => { + expect(getTerraformDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe( + TerraformDependencyTypes.unknown + ); + }); + }); +}); diff --git a/lib/manager/terraform/util.ts b/lib/manager/terraform/util.ts new file mode 100644 index 00000000000000..1d64c35398a2fe --- /dev/null +++ b/lib/manager/terraform/util.ts @@ -0,0 +1,45 @@ +import { PackageDependency } from '../common'; + +export const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; +export const githubRefMatchRegex = /github.com([/:])([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/; +export const gitTagsRefMatchRegex = /(?:git::)?((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/; + +export interface ExtractionResult { + lineNumber: number; + dependencies: PackageDependency[]; +} + +export enum TerraformDependencyTypes { + unknown = 'unknown', + module = 'module', + provider = 'provider', + required_providers = 'required_providers', +} + +export function getTerraformDependencyType( + value: string +): TerraformDependencyTypes { + switch (value) { + case 'module': { + return TerraformDependencyTypes.module; + } + case 'provider': { + return TerraformDependencyTypes.provider; + } + case 'required_providers': { + return TerraformDependencyTypes.required_providers; + } + default: { + return TerraformDependencyTypes.unknown; + } + } +} + +export function checkFileContainsDependency( + content: string, + checkList: string[] +): boolean { + return checkList.some((check) => { + return content.includes(check); + }); +} From cba9f7c8059ed60f6cfac2bb02e83ee3f7b853ad Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Mon, 29 Jun 2020 19:21:15 +0200 Subject: [PATCH 4/9] chore(terraform-provider): replace regex matcher array lookups through named lookups --- lib/manager/terraform/modules.ts | 18 +++++++++--------- lib/manager/terraform/util.ts | 4 ++-- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts index a74d6ce9dedc26..7ae9674f8be62e 100644 --- a/lib/manager/terraform/modules.ts +++ b/lib/manager/terraform/modules.ts @@ -31,11 +31,11 @@ export function analyseTerraformModule(dep: PackageDependency): void { const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source); /* eslint-disable no-param-reassign */ if (githubRefMatch) { - const depNameShort = githubRefMatch[2].replace(/\.git$/, ''); + const depNameShort = githubRefMatch.groups.project.replace(/\.git$/, ''); dep.depType = 'github'; dep.depName = 'github.com/' + depNameShort; dep.depNameShort = depNameShort; - dep.currentValue = githubRefMatch[3]; + dep.currentValue = githubRefMatch.groups.tag; dep.datasource = datasourceGithubTags.id; dep.lookupName = depNameShort; if (!isVersion(dep.currentValue)) { @@ -43,18 +43,18 @@ export function analyseTerraformModule(dep: PackageDependency): void { } } else if (gitTagsRefMatch) { dep.depType = 'gitTags'; - if (gitTagsRefMatch[2].includes('//')) { + if (gitTagsRefMatch.groups.path.includes('//')) { logger.debug('Terraform module contains subdirectory'); - dep.depName = gitTagsRefMatch[2].split('//')[0]; + dep.depName = gitTagsRefMatch.groups.path.split('//')[0]; dep.depNameShort = dep.depName.split(/\/(.+)/)[1]; - const tempLookupName = gitTagsRefMatch[1].split('//'); + const tempLookupName = gitTagsRefMatch.groups.url.split('//'); dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1]; } else { - dep.depName = gitTagsRefMatch[2].replace('.git', ''); - dep.depNameShort = gitTagsRefMatch[3].replace('.git', ''); - dep.lookupName = gitTagsRefMatch[1]; + dep.depName = gitTagsRefMatch.groups.path.replace('.git', ''); + dep.depNameShort = gitTagsRefMatch.groups.project.replace('.git', ''); + dep.lookupName = gitTagsRefMatch.groups.url; } - dep.currentValue = gitTagsRefMatch[4]; + dep.currentValue = gitTagsRefMatch.groups.tag; dep.datasource = datasourceGitTags.id; if (!isVersion(dep.currentValue)) { dep.skipReason = SkipReason.UnsupportedVersion; diff --git a/lib/manager/terraform/util.ts b/lib/manager/terraform/util.ts index 1d64c35398a2fe..e45a804d22e196 100644 --- a/lib/manager/terraform/util.ts +++ b/lib/manager/terraform/util.ts @@ -1,8 +1,8 @@ import { PackageDependency } from '../common'; export const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; -export const githubRefMatchRegex = /github.com([/:])([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/; -export const gitTagsRefMatchRegex = /(?:git::)?((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/; +export const githubRefMatchRegex = /github.com([/:])(?[^/]+\/[a-z0-9-.]+).*\?ref=(?.*)$/; +export const gitTagsRefMatchRegex = /(?:git::)?(?(?:http|https|ssh):\/\/(?:.*@)?(?.*.*\/(?.*\/.*)))\?ref=(?.*)$/; export interface ExtractionResult { lineNumber: number; From b137c2b262f7018cd71564cf9200551d72d14faf Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Mon, 29 Jun 2020 21:47:09 +0200 Subject: [PATCH 5/9] fix(terraform-provider): fix coverage --- lib/manager/terraform/extract.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 8fdc8e087657ae..84079102b8c373 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -52,7 +52,7 @@ export function extractPackageFile(content: string): PackageFile | null { ); break; } - case TerraformDependencyTypes.unknown: + /* istanbul ignore next */ default: logger.warn( `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.` From 3349d5b89feef5a97791d4fc051177988bdec109 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Mon, 29 Jun 2020 21:49:18 +0200 Subject: [PATCH 6/9] fix(terraform-provider): renamed provider.ts to providers.ts --- lib/manager/terraform/extract.ts | 5 ++++- lib/manager/terraform/modules.ts | 2 +- lib/manager/terraform/{provider.ts => providers.ts} | 0 3 files changed, 5 insertions(+), 2 deletions(-) rename lib/manager/terraform/{provider.ts => providers.ts} (100%) diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 84079102b8c373..5fab7be72dbbc8 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -1,7 +1,10 @@ import { logger } from '../../logger'; import { PackageDependency, PackageFile } from '../common'; import { analyseTerraformModule, extractTerraformModule } from './modules'; -import { analyzeTerraformProvider, extractTerraformProvider } from './provider'; +import { + analyzeTerraformProvider, + extractTerraformProvider, +} from './providers'; import { extractTerraformRequiredProviders } from './required_providers'; import { TerraformDependencyTypes, diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts index 7ae9674f8be62e..698202c80d7ce6 100644 --- a/lib/manager/terraform/modules.ts +++ b/lib/manager/terraform/modules.ts @@ -5,7 +5,7 @@ import { logger } from '../../logger'; import { SkipReason } from '../../types'; import { isVersion } from '../../versioning/hashicorp'; import { PackageDependency } from '../common'; -import { extractTerraformProvider } from './provider'; +import { extractTerraformProvider } from './providers'; import { ExtractionResult, TerraformDependencyTypes, diff --git a/lib/manager/terraform/provider.ts b/lib/manager/terraform/providers.ts similarity index 100% rename from lib/manager/terraform/provider.ts rename to lib/manager/terraform/providers.ts From 27d495a240b27a622fd7ddfeeeee457c73a26937 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Mon, 29 Jun 2020 22:00:39 +0200 Subject: [PATCH 7/9] fixup(terraform-provider): fix import --- lib/manager/terraform/required_providers.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/lib/manager/terraform/required_providers.ts b/lib/manager/terraform/required_providers.ts index 55f9ca9ad63ced..b31d685dd6bde3 100644 --- a/lib/manager/terraform/required_providers.ts +++ b/lib/manager/terraform/required_providers.ts @@ -1,6 +1,9 @@ -import { TerraformDependencyTypes } from '../../../dist/manager/terraform/extract'; import { PackageDependency } from '../common'; -import { ExtractionResult, keyValueExtractionRegex } from './util'; +import { + ExtractionResult, + TerraformDependencyTypes, + keyValueExtractionRegex, +} from './util'; export function extractTerraformRequiredProviders( startingLine: number, From 14f1955c1948a4fd30b44ee289825e467b61b5f6 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Tue, 30 Jun 2020 14:16:35 +0200 Subject: [PATCH 8/9] fixup(terraform-provider): fix code coverage --- lib/manager/terraform/extract.ts | 1 + lib/manager/terraform/modules.ts | 1 + lib/manager/terraform/providers.ts | 1 + 3 files changed, 3 insertions(+) diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index 5fab7be72dbbc8..48e469d250d24d 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -81,6 +81,7 @@ export function extractPackageFile(content: string): PackageFile | null { case TerraformDependencyTypes.module: analyseTerraformModule(dep); break; + /* istanbul ignore next */ default: } // eslint-disable-next-line no-param-reassign diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts index 698202c80d7ce6..4e1ef2cda64e19 100644 --- a/lib/manager/terraform/modules.ts +++ b/lib/manager/terraform/modules.ts @@ -73,4 +73,5 @@ export function analyseTerraformModule(dep: PackageDependency): void { logger.debug({ dep }, 'terraform dep has no source'); dep.skipReason = SkipReason.NoSource; } + /* eslint-enable no-param-reassign */ } diff --git a/lib/manager/terraform/providers.ts b/lib/manager/terraform/providers.ts index 25a4855a78a45b..d23cbd4008e1d8 100644 --- a/lib/manager/terraform/providers.ts +++ b/lib/manager/terraform/providers.ts @@ -49,4 +49,5 @@ export function analyzeTerraformProvider(dep: PackageDependency): void { if (!isValid(dep.currentValue)) { dep.skipReason = SkipReason.UnsupportedVersion; } + /* eslint-enable no-param-reassign */ } From 6fffdd0d42c89a14808ffc25213759589bd7b8f1 Mon Sep 17 00:00:00 2001 From: "sebastian.poxhofer" Date: Tue, 30 Jun 2020 14:17:16 +0200 Subject: [PATCH 9/9] fix(terraform-provider): move git(hub) regex to module scope --- lib/manager/terraform/modules.ts | 10 ++++------ lib/manager/terraform/util.ts | 2 -- 2 files changed, 4 insertions(+), 8 deletions(-) diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts index 4e1ef2cda64e19..9c84b94d0af38e 100644 --- a/lib/manager/terraform/modules.ts +++ b/lib/manager/terraform/modules.ts @@ -6,12 +6,10 @@ import { SkipReason } from '../../types'; import { isVersion } from '../../versioning/hashicorp'; import { PackageDependency } from '../common'; import { extractTerraformProvider } from './providers'; -import { - ExtractionResult, - TerraformDependencyTypes, - gitTagsRefMatchRegex, - githubRefMatchRegex, -} from './util'; +import { ExtractionResult, TerraformDependencyTypes } from './util'; + +const githubRefMatchRegex = /github.com([/:])(?[^/]+\/[a-z0-9-.]+).*\?ref=(?.*)$/; +const gitTagsRefMatchRegex = /(?:git::)?(?(?:http|https|ssh):\/\/(?:.*@)?(?.*.*\/(?.*\/.*)))\?ref=(?.*)$/; export function extractTerraformModule( startingLine: number, diff --git a/lib/manager/terraform/util.ts b/lib/manager/terraform/util.ts index e45a804d22e196..996fbae14269ff 100644 --- a/lib/manager/terraform/util.ts +++ b/lib/manager/terraform/util.ts @@ -1,8 +1,6 @@ import { PackageDependency } from '../common'; export const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; -export const githubRefMatchRegex = /github.com([/:])(?[^/]+\/[a-z0-9-.]+).*\?ref=(?.*)$/; -export const gitTagsRefMatchRegex = /(?:git::)?(?(?:http|https|ssh):\/\/(?:.*@)?(?.*.*\/(?.*\/.*)))\?ref=(?.*)$/; export interface ExtractionResult { lineNumber: number;