Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

refactor(terraform-provider): refactor manager #6637

Merged
merged 10 commits into from Jul 1, 2020
Merged

refactor(terraform-provider): refactor manager #6637

Show file tree
Hide file tree
Changes from 7 commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
0af36ae
chore(terraform-provider): replace if chain with array.some()
secustor Jun 28, 2020
914ac1d
fix(terraform-provider): make git:: prefix optional
secustor Jun 28, 2020
116af85
chore(terraform-provider): refactor terraform manager
secustor Jun 29, 2020
cba9f7c
chore(terraform-provider): replace regex matcher array lookups throug…
secustor Jun 29, 2020
b137c2b
fix(terraform-provider): fix coverage
secustor Jun 29, 2020
3349d5b
fix(terraform-provider): renamed provider.ts to providers.ts
secustor Jun 29, 2020
27d495a
fixup(terraform-provider): fix import
secustor Jun 29, 2020
14f1955
fixup(terraform-provider): fix code coverage
secustor Jun 30, 2020
6fffdd0
fix(terraform-provider): move git(hub) regex to module scope
secustor Jun 30, 2020
8ad9e47
Merge branch 'master' into terraform-provider-refactor
rarkins Jul 1, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 8 additions & 1 deletion lib/manager/terraform/__snapshots__/extract.spec.ts.snap
View file
Open in desktop
Expand Up @@ -150,7 +150,14 @@ Object {
"depType": "terraform",
"skipReason": "unsupported-version",
},
Object {},
Object {
"currentValue": "v1.0.0",
"datasource": "git-tags",
"depName": "bitbucket.com/hashicorp/example",
"depNameShort": "hashicorp/example",
"depType": "gitTags",
"lookupName": "https://bitbucket.com/hashicorp/example",
},
Object {
"currentValue": "v1.0.0",
"datasource": "git-tags",
Expand Down
38 changes: 1 addition & 37 deletions lib/manager/terraform/extract.spec.ts
View file
Open in desktop
@@ -1,9 +1,5 @@
import { readFileSync } from 'fs';
import {
TerraformDependencyTypes,
extractPackageFile,
getTerraformDependencyType,
} from './extract';
import { extractPackageFile } from './extract';

const tf1 = readFileSync('lib/manager/terraform/__fixtures__/1.tf', 'utf8');
const tf2 = `module "relative" {
Expand All @@ -26,36 +22,4 @@ describe('lib/manager/terraform/extract', () => {
expect(extractPackageFile(tf2)).toBeNull();
});
});
describe('getTerraformDependencyType()', () => {
it('returns TerraformDependencyTypes.module', () => {
expect(getTerraformDependencyType('module')).toBe(
TerraformDependencyTypes.module
);
});
it('returns TerraformDependencyTypes.provider', () => {
expect(getTerraformDependencyType('provider')).toBe(
TerraformDependencyTypes.provider
);
});
it('returns TerraformDependencyTypes.unknown', () => {
expect(getTerraformDependencyType('unknown')).toBe(
TerraformDependencyTypes.unknown
);
});
it('returns TerraformDependencyTypes.required_providers', () => {
expect(getTerraformDependencyType('required_providers')).toBe(
TerraformDependencyTypes.required_providers
);
});
it('returns TerraformDependencyTypes.unknown on empty string', () => {
expect(getTerraformDependencyType('')).toBe(
TerraformDependencyTypes.unknown
);
});
it('returns TerraformDependencyTypes.unknown on string with random chars', () => {
expect(getTerraformDependencyType('sdfsgdsfadfhfghfhgdfsdf')).toBe(
TerraformDependencyTypes.unknown
);
});
});
});
211 changes: 58 additions & 153 deletions lib/manager/terraform/extract.ts
View file
Open in desktop
@@ -1,55 +1,30 @@
import * as datasourceGitTags from '../../datasource/git-tags';
import * as datasourceGithubTags from '../../datasource/github-tags';
import * as datasourceTerraformModule from '../../datasource/terraform-module';
import * as datasourceTerraformProvider from '../../datasource/terraform-provider';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import { isValid, isVersion } from '../../versioning/hashicorp';
import { PackageDependency, PackageFile } from '../common';

export enum TerraformDependencyTypes {
unknown = 'unknown',
module = 'module',
provider = 'provider',
required_providers = 'required_providers',
}

export function getTerraformDependencyType(
value: string
): TerraformDependencyTypes {
switch (value) {
case 'module': {
return TerraformDependencyTypes.module;
}
case 'provider': {
return TerraformDependencyTypes.provider;
}
case 'required_providers': {
return TerraformDependencyTypes.required_providers;
}
default: {
return TerraformDependencyTypes.unknown;
}
}
}
import { analyseTerraformModule, extractTerraformModule } from './modules';
import {
analyzeTerraformProvider,
extractTerraformProvider,
} from './providers';
import { extractTerraformRequiredProviders } from './required_providers';
import {
TerraformDependencyTypes,
checkFileContainsDependency,
getTerraformDependencyType,
} from './util';

const dependencyBlockExtractionRegex = /^\s*(?<type>module|provider|required_providers)\s+("(?<lookupName>[^"]+)"\s+)?{\s*$/;
const keyValueExtractionRegex = /^\s*(?<key>[^\s]+)\s+=\s+"(?<value>[^"]+)"\s*$/; // extracts `exampleKey = exampleValue`
const contentCheckList = ['module "', 'provider "', 'required_providers '];

export function extractPackageFile(content: string): PackageFile | null {
logger.trace({ content }, 'terraform.extractPackageFile()');
if (
!content.includes('module "') &&
!content.includes('provider "') &&
!content.includes('required_providers ')
) {
if (!checkFileContainsDependency(content, contentCheckList)) {
return null;
}
const deps: PackageDependency[] = [];
let deps: PackageDependency[] = [];
try {
const lines = content.split('\n');
for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
let line = lines[lineNumber];
const line = lines[lineNumber];
const terraformDependency = dependencyBlockExtractionRegex.exec(line);
if (terraformDependency) {
logger.trace(
Expand All @@ -58,128 +33,58 @@ export function extractPackageFile(content: string): PackageFile | null {
const tfDepType = getTerraformDependencyType(
terraformDependency.groups.type
);

if (tfDepType === TerraformDependencyTypes.unknown) {
/* istanbul ignore next */ logger.warn(
`Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.`
);
} else if (tfDepType === TerraformDependencyTypes.required_providers) {
do {
const dep: PackageDependency = {
managerData: {
terraformDependencyType: tfDepType,
},
};

lineNumber += 1;
line = lines[lineNumber];
const kvMatch = keyValueExtractionRegex.exec(line);
if (kvMatch) {
dep.currentValue = kvMatch.groups.value;
dep.managerData.moduleName = kvMatch.groups.key;
dep.managerData.versionLine = lineNumber;
deps.push(dep);
}
} while (line.trim() !== '}');
} else {
const dep: PackageDependency = {
managerData: {
moduleName: terraformDependency.groups.lookupName,
terraformDependencyType: tfDepType,
},
};
do {
lineNumber += 1;
line = lines[lineNumber];
const kvMatch = keyValueExtractionRegex.exec(line);
if (kvMatch) {
if (kvMatch.groups.key === 'version') {
dep.currentValue = kvMatch.groups.value;
dep.managerData.versionLine = lineNumber;
} else if (kvMatch.groups.key === 'source') {
dep.managerData.source = kvMatch.groups.value;
dep.managerData.sourceLine = lineNumber;
}
}
} while (line.trim() !== '}');
deps.push(dep);
let result = null;
switch (tfDepType) {
case TerraformDependencyTypes.required_providers: {
result = extractTerraformRequiredProviders(lineNumber, lines);
break;
}
case TerraformDependencyTypes.provider: {
result = extractTerraformProvider(
lineNumber,
lines,
terraformDependency.groups.lookupName
);
break;
}
case TerraformDependencyTypes.module: {
result = extractTerraformModule(
lineNumber,
lines,
terraformDependency.groups.lookupName
);
break;
}
/* istanbul ignore next */
default:
logger.warn(
`Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.`
);
break;
}
if (result) {
lineNumber = result.lineNumber;
deps = deps.concat(result.dependencies);
result = null;
}
}
}
} catch (err) /* istanbul ignore next */ {
logger.warn({ err }, 'Error extracting buildkite plugins');
}
deps.forEach((dep) => {
if (
dep.managerData.terraformDependencyType ===
TerraformDependencyTypes.module
) {
const githubRefMatch = /github.com(\/|:)([^/]+\/[a-z0-9-.]+).*\?ref=(.*)$/.exec(
dep.managerData.source
);
const gitTagsRefMatch = /git::((?:http|https|ssh):\/\/(?:.*@)?(.*.*\/(.*\/.*)))\?ref=(.*)$/.exec(
dep.managerData.source
);
/* eslint-disable no-param-reassign */
if (githubRefMatch) {
const depNameShort = githubRefMatch[2].replace(/\.git$/, '');
dep.depType = 'github';
dep.depName = 'github.com/' + depNameShort;
dep.depNameShort = depNameShort;
dep.currentValue = githubRefMatch[3];
dep.datasource = datasourceGithubTags.id;
dep.lookupName = depNameShort;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (gitTagsRefMatch) {
dep.depType = 'gitTags';
if (gitTagsRefMatch[2].includes('//')) {
logger.debug('Terraform module contains subdirectory');
dep.depName = gitTagsRefMatch[2].split('//')[0];
dep.depNameShort = dep.depName.split(/\/(.+)/)[1];
const tempLookupName = gitTagsRefMatch[1].split('//');
dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1];
} else {
dep.depName = gitTagsRefMatch[2].replace('.git', '');
dep.depNameShort = gitTagsRefMatch[3].replace('.git', '');
dep.lookupName = gitTagsRefMatch[1];
}
dep.currentValue = gitTagsRefMatch[4];
dep.datasource = datasourceGitTags.id;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (dep.managerData.source) {
const moduleParts = dep.managerData.source.split('//')[0].split('/');
if (moduleParts[0] === '..') {
dep.skipReason = SkipReason.Local;
} else if (moduleParts.length >= 3) {
dep.depType = 'terraform';
dep.depName = moduleParts.join('/');
dep.depNameShort = dep.depName;
dep.datasource = datasourceTerraformModule.id;
}
} else {
logger.debug({ dep }, 'terraform dep has no source');
dep.skipReason = SkipReason.NoSource;
}
} else if (
dep.managerData.terraformDependencyType ===
TerraformDependencyTypes.provider ||
dep.managerData.terraformDependencyType ===
TerraformDependencyTypes.required_providers
) {
dep.depType = 'terraform';
dep.depName = dep.managerData.moduleName;
dep.depNameShort = dep.managerData.moduleName;
dep.datasource = datasourceTerraformProvider.id;
if (!isValid(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
switch (dep.managerData.terraformDependencyType) {
case TerraformDependencyTypes.required_providers:
case TerraformDependencyTypes.provider:
analyzeTerraformProvider(dep);
break;
case TerraformDependencyTypes.module:
analyseTerraformModule(dep);
break;
default:
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do we really need the default?

Copy link
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a requirement of ES-lint, without it the linting fails.

}
// eslint-disable-next-line no-param-reassign
delete dep.managerData;
/* eslint-enable no-param-reassign */
});
if (deps.some((dep) => dep.skipReason !== 'local')) {
return { deps };
Expand Down
76 changes: 76 additions & 0 deletions lib/manager/terraform/modules.ts
View file
Open in desktop
@@ -0,0 +1,76 @@
import * as datasourceGitTags from '../../datasource/git-tags';
import * as datasourceGithubTags from '../../datasource/github-tags';
import * as datasourceTerraformModule from '../../datasource/terraform-module';
import { logger } from '../../logger';
import { SkipReason } from '../../types';
import { isVersion } from '../../versioning/hashicorp';
import { PackageDependency } from '../common';
import { extractTerraformProvider } from './providers';
import {
ExtractionResult,
TerraformDependencyTypes,
gitTagsRefMatchRegex,
githubRefMatchRegex,
} from './util';

export function extractTerraformModule(
startingLine: number,
lines: string[],
moduleName: string
): ExtractionResult {
const result = extractTerraformProvider(startingLine, lines, moduleName);
result.dependencies.forEach((dep) => {
// eslint-disable-next-line no-param-reassign
dep.managerData.terraformDependencyType = TerraformDependencyTypes.module;
});
return result;
}

export function analyseTerraformModule(dep: PackageDependency): void {
const githubRefMatch = githubRefMatchRegex.exec(dep.managerData.source);
const gitTagsRefMatch = gitTagsRefMatchRegex.exec(dep.managerData.source);
/* eslint-disable no-param-reassign */
if (githubRefMatch) {
const depNameShort = githubRefMatch.groups.project.replace(/\.git$/, '');
secustor marked this conversation as resolved.
Show resolved Hide resolved
dep.depType = 'github';
dep.depName = 'github.com/' + depNameShort;
dep.depNameShort = depNameShort;
dep.currentValue = githubRefMatch.groups.tag;
dep.datasource = datasourceGithubTags.id;
dep.lookupName = depNameShort;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (gitTagsRefMatch) {
dep.depType = 'gitTags';
if (gitTagsRefMatch.groups.path.includes('//')) {
logger.debug('Terraform module contains subdirectory');
dep.depName = gitTagsRefMatch.groups.path.split('//')[0];
dep.depNameShort = dep.depName.split(/\/(.+)/)[1];
secustor marked this conversation as resolved.
Show resolved Hide resolved
const tempLookupName = gitTagsRefMatch.groups.url.split('//');
dep.lookupName = tempLookupName[0] + '//' + tempLookupName[1];
} else {
dep.depName = gitTagsRefMatch.groups.path.replace('.git', '');
dep.depNameShort = gitTagsRefMatch.groups.project.replace('.git', '');
dep.lookupName = gitTagsRefMatch.groups.url;
}
dep.currentValue = gitTagsRefMatch.groups.tag;
dep.datasource = datasourceGitTags.id;
if (!isVersion(dep.currentValue)) {
dep.skipReason = SkipReason.UnsupportedVersion;
}
} else if (dep.managerData.source) {
const moduleParts = dep.managerData.source.split('//')[0].split('/');
if (moduleParts[0] === '..') {
dep.skipReason = SkipReason.Local;
} else if (moduleParts.length >= 3) {
dep.depType = 'terraform';
dep.depName = moduleParts.join('/');
dep.depNameShort = dep.depName;
dep.datasource = datasourceTerraformModule.id;
}
} else {
logger.debug({ dep }, 'terraform dep has no source');
dep.skipReason = SkipReason.NoSource;
}
}