From e8c80fbe625acf1304ac9da8ca979e82c24e14ec Mon Sep 17 00:00:00 2001 From: Jonas Michaelis Date: Mon, 29 Jun 2020 15:11:56 +0200 Subject: [PATCH] feat(gitlabci-include): add support for 'include:local' (#6630) * feat(gitlabci-include): add support for 'include:local' * fix: lint * fix: apply suggestions from code review Co-authored-by: Michael Kriese * fix: lint * fix: includedDeps null check Co-authored-by: Michael Kriese --- .../{gitlab-ci.yaml => gitlab-ci.1.yaml} | 0 .../__fixtures__/gitlab-ci.2.yaml | 5 +++ .../__fixtures__/gitlab-ci.3.yaml | 6 +++ .../__fixtures__/include.1.yml | 5 +++ .../__fixtures__/include.2.yml | 5 +++ .../__snapshots__/extract.spec.ts.snap | 38 +++++++++++++++++ lib/manager/gitlabci-include/extract.spec.ts | 42 +++++++++++++++---- lib/manager/gitlabci-include/extract.ts | 33 ++++++++++----- 8 files changed, 115 insertions(+), 19 deletions(-) rename lib/manager/gitlabci-include/__fixtures__/{gitlab-ci.yaml => gitlab-ci.1.yaml} (100%) create mode 100644 lib/manager/gitlabci-include/__fixtures__/gitlab-ci.2.yaml create mode 100644 lib/manager/gitlabci-include/__fixtures__/gitlab-ci.3.yaml create mode 100644 lib/manager/gitlabci-include/__fixtures__/include.1.yml create mode 100644 lib/manager/gitlabci-include/__fixtures__/include.2.yml diff --git a/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.yaml b/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.1.yaml similarity index 100% rename from lib/manager/gitlabci-include/__fixtures__/gitlab-ci.yaml rename to lib/manager/gitlabci-include/__fixtures__/gitlab-ci.1.yaml diff --git a/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.2.yaml b/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.2.yaml new file mode 100644 index 00000000000000..0829ba25fcbc74 --- /dev/null +++ b/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.2.yaml @@ -0,0 +1,5 @@ +include: +- local: 'lib/manager/gitlabci-include/__fixtures__/include.1.yml' + +stages: +- test diff --git a/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.3.yaml b/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.3.yaml new file mode 100644 index 00000000000000..8607426d785cab --- /dev/null +++ b/lib/manager/gitlabci-include/__fixtures__/gitlab-ci.3.yaml @@ -0,0 +1,6 @@ +include: +- local: 'lib/manager/gitlabci-include/__fixtures__/include.1.yml' +- local: 'lib/manager/gitlabci-include/__fixtures__/include.2.yml' + +stages: +- test diff --git a/lib/manager/gitlabci-include/__fixtures__/include.1.yml b/lib/manager/gitlabci-include/__fixtures__/include.1.yml new file mode 100644 index 00000000000000..a4c0bb2233106f --- /dev/null +++ b/lib/manager/gitlabci-include/__fixtures__/include.1.yml @@ -0,0 +1,5 @@ +test: + stage: test + image: alpine:3.11 + script: + - echo test diff --git a/lib/manager/gitlabci-include/__fixtures__/include.2.yml b/lib/manager/gitlabci-include/__fixtures__/include.2.yml new file mode 100644 index 00000000000000..76e4de4fa0e754 --- /dev/null +++ b/lib/manager/gitlabci-include/__fixtures__/include.2.yml @@ -0,0 +1,5 @@ +test: + stage: test + image: node:12 + script: + - echo test diff --git a/lib/manager/gitlabci-include/__snapshots__/extract.spec.ts.snap b/lib/manager/gitlabci-include/__snapshots__/extract.spec.ts.snap index 046e4127deaf45..a0e6277d1e1afd 100644 --- a/lib/manager/gitlabci-include/__snapshots__/extract.spec.ts.snap +++ b/lib/manager/gitlabci-include/__snapshots__/extract.spec.ts.snap @@ -1,5 +1,19 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP +exports[`lib/manager/gitlabci-include/extract extractPackageFile() extracts local include block 1`] = ` +Array [ + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": "3.11", + "datasource": "docker", + "depName": "alpine", + "depType": "image", + "replaceString": "alpine:3.11", + }, +] +`; + exports[`lib/manager/gitlabci-include/extract extractPackageFile() extracts multiple include blocks 1`] = ` Array [ Object { @@ -22,3 +36,27 @@ Array [ }, ] `; + +exports[`lib/manager/gitlabci-include/extract extractPackageFile() extracts multiple local include blocks 1`] = ` +Array [ + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "currentDigest": undefined, + "currentValue": "3.11", + "datasource": "docker", + "depName": "alpine", + "depType": "image", + "replaceString": "alpine:3.11", + }, + Object { + "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}", + "commitMessageTopic": "Node.js", + "currentDigest": undefined, + "currentValue": "12", + "datasource": "docker", + "depName": "node", + "depType": "image", + "replaceString": "node:12", + }, +] +`; diff --git a/lib/manager/gitlabci-include/extract.spec.ts b/lib/manager/gitlabci-include/extract.spec.ts index 527fff5cef65e8..68c3604297044f 100644 --- a/lib/manager/gitlabci-include/extract.spec.ts +++ b/lib/manager/gitlabci-include/extract.spec.ts @@ -2,33 +2,57 @@ import fs from 'fs'; import { extractPackageFile } from './extract'; const yamlFile = fs.readFileSync( - 'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.yaml', + 'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.1.yaml', + 'utf8' +); +const yamlLocal = fs.readFileSync( + 'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.2.yaml', + 'utf8' +); + +const yamlLocalBlock = fs.readFileSync( + 'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.3.yaml', 'utf8' ); describe('lib/manager/gitlabci-include/extract', () => { describe('extractPackageFile()', () => { - it('returns null for empty', () => { + it('returns null for empty', async () => { expect( - extractPackageFile('nothing here', '.gitlab-ci.yml', {}) + await extractPackageFile('nothing here', '.gitlab-ci.yml', {}) ).toBeNull(); }); - it('extracts multiple include blocks', () => { - const res = extractPackageFile(yamlFile, '.gitlab-ci.yml', {}); + it('extracts multiple include blocks', async () => { + const res = await extractPackageFile(yamlFile, '.gitlab-ci.yml', {}); expect(res.deps).toMatchSnapshot(); expect(res.deps).toHaveLength(3); }); - it('normalizes configured endpoints', () => { + it('extracts local include block', async () => { + const res = await extractPackageFile(yamlLocal, '.gitlab-ci.yml', {}); + expect(res.deps).toMatchSnapshot(); + expect(res.deps).toHaveLength(1); + }); + it('extracts multiple local include blocks', async () => { + const res = await extractPackageFile( + yamlLocalBlock, + '.gitlab-ci.yml', + {} + ); + expect(res.deps).toMatchSnapshot(); + expect(res.deps).toHaveLength(2); + }); + it('normalizes configured endpoints', async () => { const endpoints = [ 'http://gitlab.test/api/v4', 'http://gitlab.test/api/v4/', ]; - endpoints.forEach((endpoint) => { - const res = extractPackageFile(yamlFile, '.gitlab-ci.yml', { + + for (const endpoint of endpoints) { + const res = await extractPackageFile(yamlFile, '.gitlab-ci.yml', { endpoint, }); expect(res.deps[0].registryUrls[0]).toEqual('http://gitlab.test'); - }); + } }); }); }); diff --git a/lib/manager/gitlabci-include/extract.ts b/lib/manager/gitlabci-include/extract.ts index 1e59fd662a8ea6..0dd2d75d38f6e8 100644 --- a/lib/manager/gitlabci-include/extract.ts +++ b/lib/manager/gitlabci-include/extract.ts @@ -3,16 +3,15 @@ import yaml from 'js-yaml'; import * as datasourceGitlabTags from '../../datasource/gitlab-tags'; import { logger } from '../../logger'; import { SkipReason } from '../../types'; +import { readLocalFile } from '../../util/gitfs'; import { ExtractConfig, PackageDependency, PackageFile } from '../common'; +import * as gitlabci from '../gitlabci/extract'; -function extractDepFromInclude(includeObj: { +function extractDepFromIncludeFile(includeObj: { file: any; project: string; ref: string; -}): PackageDependency | null { - if (!includeObj.file || !includeObj.project) { - return null; - } +}): PackageDependency { const dep: PackageDependency = { datasource: datasourceGitlabTags.id, depName: includeObj.project, @@ -26,22 +25,37 @@ function extractDepFromInclude(includeObj: { return dep; } -export function extractPackageFile( +async function extractDepsFromIncludeLocal(includeObj: { + local: string; +}): Promise { + const content = await readLocalFile(includeObj.local, 'utf8'); + const deps = gitlabci.extractPackageFile(content)?.deps; + return deps; +} + +export async function extractPackageFile( content: string, _packageFile: string, config: ExtractConfig -): PackageFile | null { +): Promise { const deps: PackageDependency[] = []; try { const doc = yaml.safeLoad(content, { json: true }); if (doc?.include && is.array(doc.include)) { for (const includeObj of doc.include) { - const dep = extractDepFromInclude(includeObj); - if (dep) { + if (includeObj.file && includeObj.project) { + const dep = extractDepFromIncludeFile(includeObj); if (config.endpoint) { dep.registryUrls = [config.endpoint.replace(/\/api\/v4\/?/, '')]; } deps.push(dep); + } else if (includeObj.local) { + const includedDeps = await extractDepsFromIncludeLocal(includeObj); + if (includedDeps) { + for (const includedDep of includedDeps) { + deps.push(includedDep); + } + } } } } @@ -53,7 +67,6 @@ export function extractPackageFile( logger.warn({ err }, 'Error extracting GitLab CI includes'); } } - if (!deps.length) { return null; }