feat(gitlabci-include): add support for 'include:local' (#6630)

* feat(gitlabci-include): add support for 'include:local'

* fix: lint

* fix: apply suggestions from code review

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* fix: lint

* fix: includedDeps null check

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/manager/gitlabci-include/__fixtures__/gitlab-ci.2.yaml

@@ -0,0 +1,5 @@
+include:
+- local: 'lib/manager/gitlabci-include/__fixtures__/include.1.yml'
+
+stages:
+- test

lib/manager/gitlabci-include/__fixtures__/gitlab-ci.3.yaml

@@ -0,0 +1,6 @@
+include:
+- local: 'lib/manager/gitlabci-include/__fixtures__/include.1.yml'
+- local: 'lib/manager/gitlabci-include/__fixtures__/include.2.yml'
+
+stages:
+- test

lib/manager/gitlabci-include/__fixtures__/include.1.yml

@@ -0,0 +1,5 @@
+test:
+  stage: test
+  image: alpine:3.11
+  script:
+    - echo test

lib/manager/gitlabci-include/__fixtures__/include.2.yml

@@ -0,0 +1,5 @@
+test:
+  stage: test
+  image: node:12
+  script:
+    - echo test

lib/manager/gitlabci-include/__snapshots__/extract.spec.ts.snap

@@ -1,5 +1,19 @@
 // Jest Snapshot v1, https://goo.gl/fbAQLP
 
+exports[`lib/manager/gitlabci-include/extract extractPackageFile() extracts local include block 1`] = `
+Array [
+  Object {
+    "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+    "currentDigest": undefined,
+    "currentValue": "3.11",
+    "datasource": "docker",
+    "depName": "alpine",
+    "depType": "image",
+    "replaceString": "alpine:3.11",
+  },
+]
+`;
+
 exports[`lib/manager/gitlabci-include/extract extractPackageFile() extracts multiple include blocks 1`] = `
 Array [
   Object {
@@ -22,3 +36,27 @@ Array [
   },
 ]
 `;
+
+exports[`lib/manager/gitlabci-include/extract extractPackageFile() extracts multiple local include blocks 1`] = `
+Array [
+  Object {
+    "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+    "currentDigest": undefined,
+    "currentValue": "3.11",
+    "datasource": "docker",
+    "depName": "alpine",
+    "depType": "image",
+    "replaceString": "alpine:3.11",
+  },
+  Object {
+    "autoReplaceStringTemplate": "{{depName}}{{#if newValue}}:{{newValue}}{{/if}}{{#if newDigest}}@{{newDigest}}{{/if}}",
+    "commitMessageTopic": "Node.js",
+    "currentDigest": undefined,
+    "currentValue": "12",
+    "datasource": "docker",
+    "depName": "node",
+    "depType": "image",
+    "replaceString": "node:12",
+  },
+]
+`;

lib/manager/gitlabci-include/extract.spec.ts

@@ -2,33 +2,57 @@ import fs from 'fs';
 import { extractPackageFile } from './extract';
 
 const yamlFile = fs.readFileSync(
-  'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.yaml',
+  'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.1.yaml',
+  'utf8'
+);
+const yamlLocal = fs.readFileSync(
+  'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.2.yaml',
+  'utf8'
+);
+
+const yamlLocalBlock = fs.readFileSync(
+  'lib/manager/gitlabci-include/__fixtures__/gitlab-ci.3.yaml',
   'utf8'
 );
 
 describe('lib/manager/gitlabci-include/extract', () => {
   describe('extractPackageFile()', () => {
-    it('returns null for empty', () => {
+    it('returns null for empty', async () => {
       expect(
-        extractPackageFile('nothing here', '.gitlab-ci.yml', {})
+        await extractPackageFile('nothing here', '.gitlab-ci.yml', {})
       ).toBeNull();
     });
-    it('extracts multiple include blocks', () => {
-      const res = extractPackageFile(yamlFile, '.gitlab-ci.yml', {});
+    it('extracts multiple include blocks', async () => {
+      const res = await extractPackageFile(yamlFile, '.gitlab-ci.yml', {});
       expect(res.deps).toMatchSnapshot();
       expect(res.deps).toHaveLength(3);
     });
-    it('normalizes configured endpoints', () => {
+    it('extracts local include block', async () => {
+      const res = await extractPackageFile(yamlLocal, '.gitlab-ci.yml', {});
+      expect(res.deps).toMatchSnapshot();
+      expect(res.deps).toHaveLength(1);
+    });
+    it('extracts multiple local include blocks', async () => {
+      const res = await extractPackageFile(
+        yamlLocalBlock,
+        '.gitlab-ci.yml',
+        {}
+      );
+      expect(res.deps).toMatchSnapshot();
+      expect(res.deps).toHaveLength(2);
+    });
+    it('normalizes configured endpoints', async () => {
       const endpoints = [
         'http://gitlab.test/api/v4',
         'http://gitlab.test/api/v4/',
       ];
-      endpoints.forEach((endpoint) => {
-        const res = extractPackageFile(yamlFile, '.gitlab-ci.yml', {
+
+      for (const endpoint of endpoints) {
+        const res = await extractPackageFile(yamlFile, '.gitlab-ci.yml', {
           endpoint,
         });
         expect(res.deps[0].registryUrls[0]).toEqual('http://gitlab.test');
-      });
+      }
     });
   });
 });

lib/manager/gitlabci-include/extract.ts

@@ -3,16 +3,15 @@ import yaml from 'js-yaml';
 import * as datasourceGitlabTags from '../../datasource/gitlab-tags';
 import { logger } from '../../logger';
 import { SkipReason } from '../../types';
+import { readLocalFile } from '../../util/gitfs';
 import { ExtractConfig, PackageDependency, PackageFile } from '../common';
+import * as gitlabci from '../gitlabci/extract';
 
-function extractDepFromInclude(includeObj: {
+function extractDepFromIncludeFile(includeObj: {
   file: any;
   project: string;
   ref: string;
-}): PackageDependency | null {
-  if (!includeObj.file || !includeObj.project) {
-    return null;
-  }
+}): PackageDependency {
   const dep: PackageDependency = {
     datasource: datasourceGitlabTags.id,
     depName: includeObj.project,
@@ -26,22 +25,37 @@ function extractDepFromInclude(includeObj: {
   return dep;
 }
 
-export function extractPackageFile(
+async function extractDepsFromIncludeLocal(includeObj: {
+  local: string;
+}): Promise<PackageDependency[] | null> {
+  const content = await readLocalFile(includeObj.local, 'utf8');
+  const deps = gitlabci.extractPackageFile(content)?.deps;
+  return deps;
+}
+
+export async function extractPackageFile(
   content: string,
   _packageFile: string,
   config: ExtractConfig
-): PackageFile | null {
+): Promise<PackageFile | null> {
   const deps: PackageDependency[] = [];
   try {
     const doc = yaml.safeLoad(content, { json: true });
     if (doc?.include && is.array(doc.include)) {
       for (const includeObj of doc.include) {
-        const dep = extractDepFromInclude(includeObj);
-        if (dep) {
+        if (includeObj.file && includeObj.project) {
+          const dep = extractDepFromIncludeFile(includeObj);
           if (config.endpoint) {
             dep.registryUrls = [config.endpoint.replace(/\/api\/v4\/?/, '')];
           }
           deps.push(dep);
+        } else if (includeObj.local) {
+          const includedDeps = await extractDepsFromIncludeLocal(includeObj);
+          if (includedDeps) {
+            for (const includedDep of includedDeps) {
+              deps.push(includedDep);
+            }
+          }
         }
       }
     }
@@ -53,7 +67,6 @@ export function extractPackageFile(
       logger.warn({ err }, 'Error extracting GitLab CI includes');
     }
   }
-
   if (!deps.length) {
     return null;
   }