Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
refactor(terraform-provider): refactor manager (#6637)
  • Loading branch information
secustor committed Jul 1, 2020
1 parent 45d7d40 commit b71b9f6
Show file tree
Hide file tree
Showing 8 changed files with 308 additions and 191 deletions.
9 changes: 8 additions & 1 deletion lib/manager/terraform/__snapshots__/extract.spec.ts.snap
Expand Up @@ -150,7 +150,14 @@ Object {
"depType": "terraform",
"skipReason": "unsupported-version",
},
Object {},
Object {
"currentValue": "v1.0.0",
"datasource": "git-tags",
"depName": "bitbucket.com/hashicorp/example",
"depNameShort": "hashicorp/example",
"depType": "gitTags",
"lookupName": "https://bitbucket.com/hashicorp/example",
},
Object {
"currentValue": "v1.0.0",
"datasource": "git-tags",
Expand Down
38 changes: 1 addition & 37 deletions lib/manager/terraform/extract.spec.ts
@@ -1,9 +1,5 @@
import { readFileSync } from 'fs';
import {
TerraformDependencyTypes,
extractPackageFile,
getTerraformDependencyType,
} from './extract';
import { extractPackageFile } from './extract';

const tf1 = readFileSync('lib/manager/terraform/__fixtures__/1.tf', 'utf8');
const tf2 = `module "relative" {
Expand All @@ -26,36 +22,4 @@ describe('lib/manager/terraform/extract', () => {
expect(extractPackageFile(tf2)).toBeNull();
});
});
describe('getTerraformDependencyType()', () => {
it('returns TerraformDependencyTypes.module', () => {
expect(getTerraformDependencyType('module')).toBe(
TerraformDependencyTypes.module
);
});
it('returns TerraformDependencyTypes.provider', () => {
expect(getTerraformDependencyType('provider')).toBe(
TerraformDependencyTypes.provider
);
});
it('returns TerraformDependencyTypes.unknown', () => {
expect(getTerraformDependencyType('unknown')).toBe(
TerraformDependencyTypes.unknown
);
});
it('returns TerraformDependencyTypes.required_providers', () => {
expect(getTerraformDependencyType('required_providers')).toBe(
TerraformDependencyTypes.required_providers
);
});
it('returns TerraformDependencyTypes.unknown on empty string', () => {
expect(getTerraformDependencyType('')).toBe(
TerraformDependencyTypes.unknown
);
});
it('returns TerraformDependencyTypes.unknown on string with random chars', () => {
expect(getTerraformDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe(
TerraformDependencyTypes.unknown
);
});
});
});
212 changes: 59 additions & 153 deletions lib/manager/terraform/extract.ts
@@ -1,55 +1,30 @@
import * as datasourceGitTags from '../../datasource/git-tags';
import * as datasourceGithubTags from '../../datasource/github-tags';
import * as datasourceTerraformModule from '../../datasource/terraform-module';
import * as datasourceTerraformProvider from '../../datasource/terraform-provider';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import { isValid, isVersion } from '../../versioning/hashicorp';
import { PackageDependency, PackageFile } from '../common';

export enum TerraformDependencyTypes {
unknown = 'unknown',
module = 'module',
provider = 'provider',
required_providers = 'required_providers',
}

export function getTerraformDependencyType(
value: string
): TerraformDependencyTypes {
switch (value) {
case 'module': {
return TerraformDependencyTypes.module;
}
case 'provider': {
return TerraformDependencyTypes.provider;
}
case 'required_providers': {
return TerraformDependencyTypes.required_providers;
}
default: {
return TerraformDependencyTypes.unknown;
}
}
}
import { analyseTerraformModule, extractTerraformModule } from './modules';
import {
analyzeTerraformProvider,
extractTerraformProvider,
} from './providers';
import { extractTerraformRequiredProviders } from './required_providers';
import {
TerraformDependencyTypes,
checkFileContainsDependency,
getTerraformDependencyType,
} from './util';

const dependencyBlockExtractionRegex = /^\s*(?<type>module|provider|required_providers)\s+("(?<lookupName>[^"]+)"\s+)?{\s*$/;
const keyValueExtractionRegex = /^\s*(?<key>[^\s]+)\s+=\s+"(?<value>[^"]+)"\s*$/; // extracts `exampleKey = exampleValue`
const contentCheckList = ['module "', 'provider "', 'required_providers '];

export function extractPackageFile(content: string): PackageFile | null {
logger.trace({ content }, 'terraform.extractPackageFile()');
if (
!content.includes('module "') &&
!content.includes('provider "') &&
!content.includes('required_providers ')
) {
if (!checkFileContainsDependency(content, contentCheckList)) {
return null;
}
const deps: PackageDependency[] = [];
let deps: PackageDependency[] = [];
try {
const lines = content.split('\n');
for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
let line = lines[lineNumber];
const line = lines[lineNumber];
const terraformDependency = dependencyBlockExtractionRegex.exec(line);
if (terraformDependency) {
logger.trace(
Expand All @@ -58,128 +33,59 @@ export function extractPackageFile(content: string): PackageFile | null {
const tfDepType = getTerraformDependencyType(
terraformDependency.groups.type
);

if (tfDepType === TerraformDependencyTypes.unknown) {
/* istanbul ignore next */ logger.warn(
`Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.`
);
} else if (tfDepType === TerraformDependencyTypes.required_providers) {
do {
const dep: PackageDependency = {
managerData: {
terraformDependencyType: tfDepType,
},
};

lineNumber += 1;
line = lines[lineNumber];
const kvMatch = keyValueExtractionRegex.exec(line);
if (kvMatch) {
dep.currentValue = kvMatch.groups.value;
dep.managerData.moduleName = kvMatch.groups.key;
dep.managerData.versionLine = lineNumber;
deps.push(dep);
}
} while (line.trim() !== '}');
} else {
const dep: PackageDependency = {
managerData: {
moduleName: terraformDependency.groups.lookupName,
terraformDependencyType: tfDepType,
},
};
do {
lineNumber += 1;
line = lines[lineNumber];
const kvMatch = keyValueExtractionRegex.exec(line);
if (kvMatch) {
if (kvMatch.groups.key === 'version') {
dep.currentValue = kvMatch.groups.value;
dep.managerData.versionLine = lineNumber;
} else if (kvMatch.groups.key === 'source') {
dep.managerData.source = kvMatch.groups.value;
dep.managerData.sourceLine = lineNumber;
}
}
} while (line.trim() !== '}');
deps.push(dep);
let result = null;
switch (tfDepType) {
case TerraformDependencyTypes.required_providers: {
result = extractTerraformRequiredProviders(lineNumber, lines);
break;
}
case TerraformDependencyTypes.provider: {
result = extractTerraformProvider(
lineNumber,
lines,
terraformDependency.groups.lookupName
);
break;
}
case TerraformDependencyTypes.module: {
result = extractTerraformModule(
lineNumber,
lines,
terraformDependency.groups.lookupName
);
break;
}
/* istanbul ignore next */
default:
logger.warn(
`Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.`
);
break;
}
if (result) {
lineNumber = result.lineNumber;
deps = deps.concat(result.dependencies);
result = null;
}
}
}
} catch (err) /* istanbul ignore next */ {
logger.warn({ err }, 'Error extracting buildkite plugins');
}
deps.forEach((dep) => {
if (
dep.managerData.terraformDependencyType ===
TerraformDependencyTypes.module
) {
const githubRefMatch = /github.com(\/|:)([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/.exec(
dep.managerData.source
);
const gitTagsRefMatch = /git::((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/.exec(
dep.managerData.source
);
/* eslint-disable no-param-reassign */
if (githubRefMatch) {
const depNameShort = githubRefMatch[2].replace(/\.git$/, '');
dep.depType = 'github';
dep.depName = 'github.com/' + depNameShort;
dep.depNameShort = depNameShort;
dep.currentValue = githubRefMatch[3];
dep.datasource = datasourceGithubTags.id;
dep.lookupName = depNameShort;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (gitTagsRefMatch) {
dep.depType = 'gitTags';
if (gitTagsRefMatch[2].includes('//')) {
logger.debug('Terraform module contains subdirectory');
dep.depName = gitTagsRefMatch[2].split('//')[0];
dep.depNameShort = dep.depName.split(/\/(.+)/)[1];
const tempLookupName = gitTagsRefMatch[1].split('//');
dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1];
} else {
dep.depName = gitTagsRefMatch[2].replace('.git', '');
dep.depNameShort = gitTagsRefMatch[3].replace('.git', '');
dep.lookupName = gitTagsRefMatch[1];
}
dep.currentValue = gitTagsRefMatch[4];
dep.datasource = datasourceGitTags.id;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (dep.managerData.source) {
const moduleParts = dep.managerData.source.split('//')[0].split('/');
if (moduleParts[0] === '..') {
dep.skipReason = SkipReason.Local;
} else if (moduleParts.length >= 3) {
dep.depType = 'terraform';
dep.depName = moduleParts.join('/');
dep.depNameShort = dep.depName;
dep.datasource = datasourceTerraformModule.id;
}
} else {
logger.debug({ dep }, 'terraform dep has no source');
dep.skipReason = SkipReason.NoSource;
}
} else if (
dep.managerData.terraformDependencyType ===
TerraformDependencyTypes.provider ||
dep.managerData.terraformDependencyType ===
TerraformDependencyTypes.required_providers
) {
dep.depType = 'terraform';
dep.depName = dep.managerData.moduleName;
dep.depNameShort = dep.managerData.moduleName;
dep.datasource = datasourceTerraformProvider.id;
if (!isValid(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
switch (dep.managerData.terraformDependencyType) {
case TerraformDependencyTypes.required_providers:
case TerraformDependencyTypes.provider:
analyzeTerraformProvider(dep);
break;
case TerraformDependencyTypes.module:
analyseTerraformModule(dep);
break;
/* istanbul ignore next */
default:
}
// eslint-disable-next-line no-param-reassign
delete dep.managerData;
/* eslint-enable no-param-reassign */
});
if (deps.some((dep) => dep.skipReason !== 'local')) {
return { deps };
Expand Down
75 changes: 75 additions & 0 deletions lib/manager/terraform/modules.ts
@@ -0,0 +1,75 @@
import * as datasourceGitTags from '../../datasource/git-tags';
import * as datasourceGithubTags from '../../datasource/github-tags';
import * as datasourceTerraformModule from '../../datasource/terraform-module';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import { isVersion } from '../../versioning/hashicorp';
import { PackageDependency } from '../common';
import { extractTerraformProvider } from './providers';
import { ExtractionResult, TerraformDependencyTypes } from './util';

const githubRefMatchRegex = /github.com([/:])(?<project>[^/]+\/[a-z0-9-.]+).*\?ref=(?<tag>.*)$/;
const gitTagsRefMatchRegex = /(?:git::)?(?<url>(?:http|https|ssh):\/\/(?:.*@)?(?<path>.*.*\/(?<project>.*\/.*)))\?ref=(?<tag>.*)$/;

export function extractTerraformModule(
startingLine: number,
lines: string[],
moduleName: string
): ExtractionResult {
const result = extractTerraformProvider(startingLine, lines, moduleName);
result.dependencies.forEach((dep) => {
// eslint-disable-next-line no-param-reassign
dep.managerData.terraformDependencyType = TerraformDependencyTypes.module;
});
return result;
}

export function analyseTerraformModule(dep: PackageDependency): void {
const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source);
const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source);
/* eslint-disable no-param-reassign */
if (githubRefMatch) {
const depNameShort = githubRefMatch.groups.project.replace(/\.git$/, '');
dep.depType = 'github';
dep.depName = 'github.com/' + depNameShort;
dep.depNameShort = depNameShort;
dep.currentValue = githubRefMatch.groups.tag;
dep.datasource = datasourceGithubTags.id;
dep.lookupName = depNameShort;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (gitTagsRefMatch) {
dep.depType = 'gitTags';
if (gitTagsRefMatch.groups.path.includes('//')) {
logger.debug('Terraform module contains subdirectory');
dep.depName = gitTagsRefMatch.groups.path.split('//')[0];
dep.depNameShort = dep.depName.split(/\/(.+)/)[1];
const tempLookupName = gitTagsRefMatch.groups.url.split('//');
dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1];
} else {
dep.depName = gitTagsRefMatch.groups.path.replace('.git', '');
dep.depNameShort = gitTagsRefMatch.groups.project.replace('.git', '');
dep.lookupName = gitTagsRefMatch.groups.url;
}
dep.currentValue = gitTagsRefMatch.groups.tag;
dep.datasource = datasourceGitTags.id;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (dep.managerData.source) {
const moduleParts = dep.managerData.source.split('//')[0].split('/');
if (moduleParts[0] === '..') {
dep.skipReason = SkipReason.Local;
} else if (moduleParts.length >= 3) {
dep.depType = 'terraform';
dep.depName = moduleParts.join('/');
dep.depNameShort = dep.depName;
dep.datasource = datasourceTerraformModule.id;
}
} else {
logger.debug({ dep }, 'terraform dep has no source');
dep.skipReason = SkipReason.NoSource;
}
/* eslint-enable no-param-reassign */
}

0 comments on commit b71b9f6

Please sign in to comment.