From b71b9f6f5d03180c72bbf3c6a46a0d37f4c6f051 Mon Sep 17 00:00:00 2001 From: Sebastian Poxhofer Date: Wed, 1 Jul 2020 17:17:40 +0200 Subject: [PATCH] refactor(terraform-provider): refactor manager (#6637) --- .../__snapshots__/extract.spec.ts.snap | 9 +- lib/manager/terraform/extract.spec.ts | 38 +--- lib/manager/terraform/extract.ts | 212 +++++------------- lib/manager/terraform/modules.ts | 75 +++++++ lib/manager/terraform/providers.ts | 53 +++++ lib/manager/terraform/required_providers.ts | 33 +++ lib/manager/terraform/util.spec.ts | 36 +++ lib/manager/terraform/util.ts | 43 ++++ 8 files changed, 308 insertions(+), 191 deletions(-) create mode 100644 lib/manager/terraform/modules.ts create mode 100644 lib/manager/terraform/providers.ts create mode 100644 lib/manager/terraform/required_providers.ts create mode 100644 lib/manager/terraform/util.spec.ts create mode 100644 lib/manager/terraform/util.ts diff --git a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap index 195e5888bf9b42..e61646a6faa8bd 100644 --- a/lib/manager/terraform/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/terraform/__snapshots__/extract.spec.ts.snap @@ -150,7 +150,14 @@ Object { "depType": "terraform", "skipReason": "unsupported-version", }, - Object {}, + Object { + "currentValue": "v1.0.0", + "datasource": "git-tags", + "depName": "bitbucket.com/hashicorp/example", + "depNameShort": "hashicorp/example", + "depType": "gitTags", + "lookupName": "https://bitbucket.com/hashicorp/example", + }, Object { "currentValue": "v1.0.0", "datasource": "git-tags", diff --git a/lib/manager/terraform/extract.spec.ts b/lib/manager/terraform/extract.spec.ts index 9b3e94c93504f8..5e3fba602b7475 100644 --- a/lib/manager/terraform/extract.spec.ts +++ b/lib/manager/terraform/extract.spec.ts @@ -1,9 +1,5 @@ import { readFileSync } from 'fs'; -import { - TerraformDependencyTypes, - extractPackageFile, - getTerraformDependencyType, -} from './extract'; +import { extractPackageFile } from './extract'; const tf1 = readFileSync('lib/manager/terraform/__fixtures__/1.tf', 'utf8'); const tf2 = `module "relative" { @@ -26,36 +22,4 @@ describe('lib/manager/terraform/extract', () => { expect(extractPackageFile(tf2)).toBeNull(); }); }); - describe('getTerraformDependencyType()', () => { - it('returns TerraformDependencyTypes.module', () => { - expect(getTerraformDependencyType('module')).toBe( - TerraformDependencyTypes.module - ); - }); - it('returns TerraformDependencyTypes.provider', () => { - expect(getTerraformDependencyType('provider')).toBe( - TerraformDependencyTypes.provider - ); - }); - it('returns TerraformDependencyTypes.unknown', () => { - expect(getTerraformDependencyType('unknown')).toBe( - TerraformDependencyTypes.unknown - ); - }); - it('returns TerraformDependencyTypes.required_providers', () => { - expect(getTerraformDependencyType('required_providers')).toBe( - TerraformDependencyTypes.required_providers - ); - }); - it('returns TerraformDependencyTypes.unknown on empty string', () => { - expect(getTerraformDependencyType('')).toBe( - TerraformDependencyTypes.unknown - ); - }); - it('returns TerraformDependencyTypes.unknown on string with random chars', () => { - expect(getTerraformDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe( - TerraformDependencyTypes.unknown - ); - }); - }); }); diff --git a/lib/manager/terraform/extract.ts b/lib/manager/terraform/extract.ts index dad95431b6ee61..48e469d250d24d 100644 --- a/lib/manager/terraform/extract.ts +++ b/lib/manager/terraform/extract.ts @@ -1,55 +1,30 @@ -import * as datasourceGitTags from '../../datasource/git-tags'; -import * as datasourceGithubTags from '../../datasource/github-tags'; -import * as datasourceTerraformModule from '../../datasource/terraform-module'; -import * as datasourceTerraformProvider from '../../datasource/terraform-provider'; import { logger } from '../../logger'; -import { SkipReason } from '../../types'; -import { isValid, isVersion } from '../../versioning/hashicorp'; import { PackageDependency, PackageFile } from '../common'; - -export enum TerraformDependencyTypes { - unknown = 'unknown', - module = 'module', - provider = 'provider', - required_providers = 'required_providers', -} - -export function getTerraformDependencyType( - value: string -): TerraformDependencyTypes { - switch (value) { - case 'module': { - return TerraformDependencyTypes.module; - } - case 'provider': { - return TerraformDependencyTypes.provider; - } - case 'required_providers': { - return TerraformDependencyTypes.required_providers; - } - default: { - return TerraformDependencyTypes.unknown; - } - } -} +import { analyseTerraformModule, extractTerraformModule } from './modules'; +import { + analyzeTerraformProvider, + extractTerraformProvider, +} from './providers'; +import { extractTerraformRequiredProviders } from './required_providers'; +import { + TerraformDependencyTypes, + checkFileContainsDependency, + getTerraformDependencyType, +} from './util'; const dependencyBlockExtractionRegex = /^\s*(?module|provider|required_providers)\s+("(?[^"]+)"\s+)?{\s*$/; -const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; // extracts `exampleKey = exampleValue` +const contentCheckList = ['module "', 'provider "', 'required_providers ']; export function extractPackageFile(content: string): PackageFile | null { logger.trace({ content }, 'terraform.extractPackageFile()'); - if ( - !content.includes('module "') && - !content.includes('provider "') && - !content.includes('required_providers ') - ) { + if (!checkFileContainsDependency(content, contentCheckList)) { return null; } - const deps: PackageDependency[] = []; + let deps: PackageDependency[] = []; try { const lines = content.split('\n'); for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) { - let line = lines[lineNumber]; + const line = lines[lineNumber]; const terraformDependency = dependencyBlockExtractionRegex.exec(line); if (terraformDependency) { logger.trace( @@ -58,51 +33,39 @@ export function extractPackageFile(content: string): PackageFile | null { const tfDepType = getTerraformDependencyType( terraformDependency.groups.type ); - - if (tfDepType === TerraformDependencyTypes.unknown) { - /* istanbul ignore next */ logger.warn( - `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.` - ); - } else if (tfDepType === TerraformDependencyTypes.required_providers) { - do { - const dep: PackageDependency = { - managerData: { - terraformDependencyType: tfDepType, - }, - }; - - lineNumber += 1; - line = lines[lineNumber]; - const kvMatch = keyValueExtractionRegex.exec(line); - if (kvMatch) { - dep.currentValue = kvMatch.groups.value; - dep.managerData.moduleName = kvMatch.groups.key; - dep.managerData.versionLine = lineNumber; - deps.push(dep); - } - } while (line.trim() !== '}'); - } else { - const dep: PackageDependency = { - managerData: { - moduleName: terraformDependency.groups.lookupName, - terraformDependencyType: tfDepType, - }, - }; - do { - lineNumber += 1; - line = lines[lineNumber]; - const kvMatch = keyValueExtractionRegex.exec(line); - if (kvMatch) { - if (kvMatch.groups.key === 'version') { - dep.currentValue = kvMatch.groups.value; - dep.managerData.versionLine = lineNumber; - } else if (kvMatch.groups.key === 'source') { - dep.managerData.source = kvMatch.groups.value; - dep.managerData.sourceLine = lineNumber; - } - } - } while (line.trim() !== '}'); - deps.push(dep); + let result = null; + switch (tfDepType) { + case TerraformDependencyTypes.required_providers: { + result = extractTerraformRequiredProviders(lineNumber, lines); + break; + } + case TerraformDependencyTypes.provider: { + result = extractTerraformProvider( + lineNumber, + lines, + terraformDependency.groups.lookupName + ); + break; + } + case TerraformDependencyTypes.module: { + result = extractTerraformModule( + lineNumber, + lines, + terraformDependency.groups.lookupName + ); + break; + } + /* istanbul ignore next */ + default: + logger.warn( + `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.` + ); + break; + } + if (result) { + lineNumber = result.lineNumber; + deps = deps.concat(result.dependencies); + result = null; } } } @@ -110,76 +73,19 @@ export function extractPackageFile(content: string): PackageFile | null { logger.warn({ err }, 'Error extracting buildkite plugins'); } deps.forEach((dep) => { - if ( - dep.managerData.terraformDependencyType === - TerraformDependencyTypes.module - ) { - const githubRefMatch = /github.com(\/|:)([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/.exec( - dep.managerData.source - ); - const gitTagsRefMatch = /git::((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/.exec( - dep.managerData.source - ); - /* eslint-disable no-param-reassign */ - if (githubRefMatch) { - const depNameShort = githubRefMatch[2].replace(/\.git$/, ''); - dep.depType = 'github'; - dep.depName = 'github.com/' + depNameShort; - dep.depNameShort = depNameShort; - dep.currentValue = githubRefMatch[3]; - dep.datasource = datasourceGithubTags.id; - dep.lookupName = depNameShort; - if (!isVersion(dep.currentValue)) { - dep.skipReason = SkipReason.UnsupportedVersion; - } - } else if (gitTagsRefMatch) { - dep.depType = 'gitTags'; - if (gitTagsRefMatch[2].includes('//')) { - logger.debug('Terraform module contains subdirectory'); - dep.depName = gitTagsRefMatch[2].split('//')[0]; - dep.depNameShort = dep.depName.split(/\/(.+)/)[1]; - const tempLookupName = gitTagsRefMatch[1].split('//'); - dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1]; - } else { - dep.depName = gitTagsRefMatch[2].replace('.git', ''); - dep.depNameShort = gitTagsRefMatch[3].replace('.git', ''); - dep.lookupName = gitTagsRefMatch[1]; - } - dep.currentValue = gitTagsRefMatch[4]; - dep.datasource = datasourceGitTags.id; - if (!isVersion(dep.currentValue)) { - dep.skipReason = SkipReason.UnsupportedVersion; - } - } else if (dep.managerData.source) { - const moduleParts = dep.managerData.source.split('//')[0].split('/'); - if (moduleParts[0] === '..') { - dep.skipReason = SkipReason.Local; - } else if (moduleParts.length >= 3) { - dep.depType = 'terraform'; - dep.depName = moduleParts.join('/'); - dep.depNameShort = dep.depName; - dep.datasource = datasourceTerraformModule.id; - } - } else { - logger.debug({ dep }, 'terraform dep has no source'); - dep.skipReason = SkipReason.NoSource; - } - } else if ( - dep.managerData.terraformDependencyType === - TerraformDependencyTypes.provider || - dep.managerData.terraformDependencyType === - TerraformDependencyTypes.required_providers - ) { - dep.depType = 'terraform'; - dep.depName = dep.managerData.moduleName; - dep.depNameShort = dep.managerData.moduleName; - dep.datasource = datasourceTerraformProvider.id; - if (!isValid(dep.currentValue)) { - dep.skipReason = SkipReason.UnsupportedVersion; - } + switch (dep.managerData.terraformDependencyType) { + case TerraformDependencyTypes.required_providers: + case TerraformDependencyTypes.provider: + analyzeTerraformProvider(dep); + break; + case TerraformDependencyTypes.module: + analyseTerraformModule(dep); + break; + /* istanbul ignore next */ + default: } + // eslint-disable-next-line no-param-reassign delete dep.managerData; - /* eslint-enable no-param-reassign */ }); if (deps.some((dep) => dep.skipReason !== 'local')) { return { deps }; diff --git a/lib/manager/terraform/modules.ts b/lib/manager/terraform/modules.ts new file mode 100644 index 00000000000000..9c84b94d0af38e --- /dev/null +++ b/lib/manager/terraform/modules.ts @@ -0,0 +1,75 @@ +import * as datasourceGitTags from '../../datasource/git-tags'; +import * as datasourceGithubTags from '../../datasource/github-tags'; +import * as datasourceTerraformModule from '../../datasource/terraform-module'; +import { logger } from '../../logger'; +import { SkipReason } from '../../types'; +import { isVersion } from '../../versioning/hashicorp'; +import { PackageDependency } from '../common'; +import { extractTerraformProvider } from './providers'; +import { ExtractionResult, TerraformDependencyTypes } from './util'; + +const githubRefMatchRegex = /github.com([/:])(?[^/]+\/[a-z0-9-.]+).*\?ref=(?.*)$/; +const gitTagsRefMatchRegex = /(?:git::)?(?(?:http|https|ssh):\/\/(?:.*@)?(?.*.*\/(?.*\/.*)))\?ref=(?.*)$/; + +export function extractTerraformModule( + startingLine: number, + lines: string[], + moduleName: string +): ExtractionResult { + const result = extractTerraformProvider(startingLine, lines, moduleName); + result.dependencies.forEach((dep) => { + // eslint-disable-next-line no-param-reassign + dep.managerData.terraformDependencyType = TerraformDependencyTypes.module; + }); + return result; +} + +export function analyseTerraformModule(dep: PackageDependency): void { + const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source); + const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source); + /* eslint-disable no-param-reassign */ + if (githubRefMatch) { + const depNameShort = githubRefMatch.groups.project.replace(/\.git$/, ''); + dep.depType = 'github'; + dep.depName = 'github.com/' + depNameShort; + dep.depNameShort = depNameShort; + dep.currentValue = githubRefMatch.groups.tag; + dep.datasource = datasourceGithubTags.id; + dep.lookupName = depNameShort; + if (!isVersion(dep.currentValue)) { + dep.skipReason = SkipReason.UnsupportedVersion; + } + } else if (gitTagsRefMatch) { + dep.depType = 'gitTags'; + if (gitTagsRefMatch.groups.path.includes('//')) { + logger.debug('Terraform module contains subdirectory'); + dep.depName = gitTagsRefMatch.groups.path.split('//')[0]; + dep.depNameShort = dep.depName.split(/\/(.+)/)[1]; + const tempLookupName = gitTagsRefMatch.groups.url.split('//'); + dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1]; + } else { + dep.depName = gitTagsRefMatch.groups.path.replace('.git', ''); + dep.depNameShort = gitTagsRefMatch.groups.project.replace('.git', ''); + dep.lookupName = gitTagsRefMatch.groups.url; + } + dep.currentValue = gitTagsRefMatch.groups.tag; + dep.datasource = datasourceGitTags.id; + if (!isVersion(dep.currentValue)) { + dep.skipReason = SkipReason.UnsupportedVersion; + } + } else if (dep.managerData.source) { + const moduleParts = dep.managerData.source.split('//')[0].split('/'); + if (moduleParts[0] === '..') { + dep.skipReason = SkipReason.Local; + } else if (moduleParts.length >= 3) { + dep.depType = 'terraform'; + dep.depName = moduleParts.join('/'); + dep.depNameShort = dep.depName; + dep.datasource = datasourceTerraformModule.id; + } + } else { + logger.debug({ dep }, 'terraform dep has no source'); + dep.skipReason = SkipReason.NoSource; + } + /* eslint-enable no-param-reassign */ +} diff --git a/lib/manager/terraform/providers.ts b/lib/manager/terraform/providers.ts new file mode 100644 index 00000000000000..d23cbd4008e1d8 --- /dev/null +++ b/lib/manager/terraform/providers.ts @@ -0,0 +1,53 @@ +import * as datasourceTerraformProvider from '../../datasource/terraform-provider'; +import { SkipReason } from '../../types'; +import { isValid } from '../../versioning/hashicorp'; +import { PackageDependency } from '../common'; +import { + ExtractionResult, + TerraformDependencyTypes, + keyValueExtractionRegex, +} from './util'; + +export function extractTerraformProvider( + startingLine: number, + lines: string[], + moduleName: string +): ExtractionResult { + let lineNumber = startingLine; + let line: string; + const deps: PackageDependency[] = []; + const dep: PackageDependency = { + managerData: { + moduleName, + terraformDependencyType: TerraformDependencyTypes.provider, + }, + }; + do { + lineNumber += 1; + line = lines[lineNumber]; + const kvMatch = keyValueExtractionRegex.exec(line); + if (kvMatch) { + if (kvMatch.groups.key === 'version') { + dep.currentValue = kvMatch.groups.value; + dep.managerData.versionLine = lineNumber; + } else if (kvMatch.groups.key === 'source') { + dep.managerData.source = kvMatch.groups.value; + dep.managerData.sourceLine = lineNumber; + } + } + } while (line.trim() !== '}'); + deps.push(dep); + return { lineNumber, dependencies: deps }; +} + +export function analyzeTerraformProvider(dep: PackageDependency): void { + /* eslint-disable no-param-reassign */ + dep.depType = 'terraform'; + dep.depName = dep.managerData.moduleName; + dep.depNameShort = dep.managerData.moduleName; + dep.datasource = datasourceTerraformProvider.id; + if (!isValid(dep.currentValue)) { + dep.skipReason = SkipReason.UnsupportedVersion; + } + /* eslint-enable no-param-reassign */ +} diff --git a/lib/manager/terraform/required_providers.ts b/lib/manager/terraform/required_providers.ts new file mode 100644 index 00000000000000..b31d685dd6bde3 --- /dev/null +++ b/lib/manager/terraform/required_providers.ts @@ -0,0 +1,33 @@ +import { PackageDependency } from '../common'; +import { + ExtractionResult, + TerraformDependencyTypes, + keyValueExtractionRegex, +} from './util'; + +export function extractTerraformRequiredProviders( + startingLine: number, + lines: string[] +): ExtractionResult { + let lineNumber = startingLine; + let line: string; + const deps: PackageDependency[] = []; + do { + const dep: PackageDependency = { + managerData: { + terraformDependencyType: TerraformDependencyTypes.required_providers, + }, + }; + + lineNumber += 1; + line = lines[lineNumber]; + const kvMatch = keyValueExtractionRegex.exec(line); + if (kvMatch) { + dep.currentValue = kvMatch.groups.value; + dep.managerData.moduleName = kvMatch.groups.key; + dep.managerData.versionLine = lineNumber; + deps.push(dep); + } + } while (line.trim() !== '}'); + return { lineNumber, dependencies: deps }; +} diff --git a/lib/manager/terraform/util.spec.ts b/lib/manager/terraform/util.spec.ts new file mode 100644 index 00000000000000..cb1de0af4c1379 --- /dev/null +++ b/lib/manager/terraform/util.spec.ts @@ -0,0 +1,36 @@ +import { TerraformDependencyTypes, getTerraformDependencyType } from './util'; + +describe('lib/manager/terraform/extract', () => { + describe('getTerraformDependencyType()', () => { + it('returns TerraformDependencyTypes.module', () => { + expect(getTerraformDependencyType('module')).toBe( + TerraformDependencyTypes.module + ); + }); + it('returns TerraformDependencyTypes.provider', () => { + expect(getTerraformDependencyType('provider')).toBe( + TerraformDependencyTypes.provider + ); + }); + it('returns TerraformDependencyTypes.unknown', () => { + expect(getTerraformDependencyType('unknown')).toBe( + TerraformDependencyTypes.unknown + ); + }); + it('returns TerraformDependencyTypes.required_providers', () => { + expect(getTerraformDependencyType('required_providers')).toBe( + TerraformDependencyTypes.required_providers + ); + }); + it('returns TerraformDependencyTypes.unknown on empty string', () => { + expect(getTerraformDependencyType('')).toBe( + TerraformDependencyTypes.unknown + ); + }); + it('returns TerraformDependencyTypes.unknown on string with random chars', () => { + expect(getTerraformDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe( + TerraformDependencyTypes.unknown + ); + }); + }); +}); diff --git a/lib/manager/terraform/util.ts b/lib/manager/terraform/util.ts new file mode 100644 index 00000000000000..996fbae14269ff --- /dev/null +++ b/lib/manager/terraform/util.ts @@ -0,0 +1,43 @@ +import { PackageDependency } from '../common'; + +export const keyValueExtractionRegex = /^\s*(?[^\s]+)\s+=\s+"(?[^"]+)"\s*$/; + +export interface ExtractionResult { + lineNumber: number; + dependencies: PackageDependency[]; +} + +export enum TerraformDependencyTypes { + unknown = 'unknown', + module = 'module', + provider = 'provider', + required_providers = 'required_providers', +} + +export function getTerraformDependencyType( + value: string +): TerraformDependencyTypes { + switch (value) { + case 'module': { + return TerraformDependencyTypes.module; + } + case 'provider': { + return TerraformDependencyTypes.provider; + } + case 'required_providers': { + return TerraformDependencyTypes.required_providers; + } + default: { + return TerraformDependencyTypes.unknown; + } + } +} + +export function checkFileContainsDependency( + content: string, + checkList: string[] +): boolean { + return checkList.some((check) => { + return content.includes(check); + }); +}