feat(gitlab-ci): ref add logic for updating non top level includes (#…

…16819)

Co-authored-by: Fred Rondina <fred.rondina@daveramsey.com>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>

lib/modules/manager/gitlabci-include/__fixtures__/gitlab-ci.4.yaml

@@ -0,0 +1,13 @@
+---
+include:
+- project: mikebryant/include-source-example
+  file: /template.yaml
+  ref: 1.0.0
+
+trigger-my-job:
+  extends: .extend-trigger-job
+  trigger:
+    include:
+      - project: mikebryant/include-source-example
+        file: /template.yaml
+        ref: master

lib/modules/manager/gitlabci-include/common.spec.ts

@@ -0,0 +1,60 @@
+import { load } from 'js-yaml';
+import { Fixtures } from '../../../../test/fixtures';
+import type { GitlabPipeline } from '../gitlabci/types';
+import { replaceReferenceTags } from '../gitlabci/utils';
+import {
+  filterIncludeFromGitlabPipeline,
+  isGitlabIncludeLocal,
+  isGitlabIncludeProject,
+  isNonEmptyObject,
+} from './common';
+
+const yamlFileMultiConfig = Fixtures.get('gitlab-ci.1.yaml');
+const pipeline = load(
+  replaceReferenceTags(yamlFileMultiConfig)
+) as GitlabPipeline;
+const includeLocal = { local: 'something' };
+const includeProject = { project: 'something' };
+
+describe('modules/manager/gitlabci-include/common', () => {
+  describe('filterIncludeFromGitlabPipeline()', () => {
+    it('returns GitlabPipeline without top level include key', () => {
+      expect(pipeline).toHaveProperty('include');
+      const filtered_pipeline = filterIncludeFromGitlabPipeline(pipeline);
+      expect(filtered_pipeline).not.toHaveProperty('include');
+      expect(filtered_pipeline).toEqual({
+        script: [null, null],
+      });
+    });
+  });
+
+  describe('isGitlabIncludeLocal()', () => {
+    it('returns true if GitlabInclude is GitlabIncludeLocal', () => {
+      expect(isGitlabIncludeLocal(includeLocal)).toBe(true);
+    });
+
+    it('returns false if GitlabInclude is not GitlabIncludeLocal', () => {
+      expect(isGitlabIncludeLocal(includeProject)).toBe(false);
+    });
+  });
+
+  describe('isGitlabIncludeProject()', () => {
+    it('returns true if GitlabInclude is GitlabIncludeProject', () => {
+      expect(isGitlabIncludeProject(includeProject)).toBe(true);
+    });
+
+    it('returns false if GitlabInclude is not GitlabIncludeProject', () => {
+      expect(isGitlabIncludeProject(includeLocal)).toBe(false);
+    });
+  });
+
+  describe('isNonEmptyObject()', () => {
+    it('returns true if not empty', () => {
+      expect(isNonEmptyObject({ attribute1: 1 })).toBe(true);
+    });
+
+    it('returns false if empty', () => {
+      expect(isNonEmptyObject({})).toBe(false);
+    });
+  });
+});

lib/modules/manager/gitlabci-include/common.ts

@@ -0,0 +1,34 @@
+import is from '@sindresorhus/is';
+import type {
+  GitlabInclude,
+  GitlabIncludeLocal,
+  GitlabIncludeProject,
+  GitlabPipeline,
+} from '../gitlabci/types';
+
+export function isNonEmptyObject(obj: any): boolean {
+  return is.object(obj) && Object.keys(obj).length !== 0;
+}
+
+export function filterIncludeFromGitlabPipeline(
+  pipeline: GitlabPipeline
+): GitlabPipeline {
+  const pipeline_without_include = {} as GitlabPipeline;
+  for (const key of Object.keys(pipeline).filter((key) => key !== 'include')) {
+    const pipeline_key = key as keyof typeof pipeline;
+    pipeline_without_include[pipeline_key] = pipeline[pipeline_key];
+  }
+  return pipeline_without_include;
+}
+
+export function isGitlabIncludeProject(
+  include: GitlabInclude
+): include is GitlabIncludeProject {
+  return !is.undefined((include as GitlabIncludeProject).project);
+}
+
+export function isGitlabIncludeLocal(
+  include: GitlabInclude
+): include is GitlabIncludeLocal {
+  return !is.undefined((include as GitlabIncludeLocal).local);
+}

lib/modules/manager/gitlabci-include/extract.spec.ts

@@ -5,6 +5,7 @@ import { extractPackageFile } from '.';
 const yamlFileMultiConfig = Fixtures.get('gitlab-ci.1.yaml');
 const yamlFileSingleConfig = Fixtures.get('gitlab-ci.2.yaml');
 const yamlWithEmptyIncludeConfig = Fixtures.get('gitlab-ci.3.yaml');
+const yamlWithTriggerRef = Fixtures.get('gitlab-ci.4.yaml');
 
 describe('modules/manager/gitlabci-include/extract', () => {
   describe('extractPackageFile()', () => {
@@ -29,6 +30,32 @@ describe('modules/manager/gitlabci-include/extract', () => {
       expect(res?.deps).toHaveLength(3);
     });
 
+    it('extracts multiple embedded include blocks', () => {
+      const res = extractPackageFile(yamlWithTriggerRef);
+      expect(res?.deps).toHaveLength(2);
+      expect(res?.deps).toMatchObject([
+        {
+          currentValue: 'master',
+          datasource: 'gitlab-tags',
+          depName: 'mikebryant/include-source-example',
+        },
+        {
+          currentValue: '1.0.0',
+          datasource: 'gitlab-tags',
+          depName: 'mikebryant/include-source-example',
+        },
+      ]);
+    });
+
+    it('ignores includes without project and file keys', () => {
+      const includeWithoutProjectRef = `include:
+      - 'https://gitlab.com/mikebryant/include-source-example.yml'
+      - remote: 'https://gitlab.com/mikebryant/include-source-example.yml'
+      - local: mikebryant/include-source-example`;
+      const res = extractPackageFile(includeWithoutProjectRef);
+      expect(res).toBeNull();
+    });
+
     it('normalizes configured endpoints', () => {
       const endpoints = [
         'http://gitlab.test/api/v4',

lib/modules/manager/gitlabci-include/extract.ts

@@ -4,14 +4,22 @@ import { GlobalConfig } from '../../../config/global';
 import { logger } from '../../../logger';
 import { regEx } from '../../../util/regex';
 import { GitlabTagsDatasource } from '../../datasource/gitlab-tags';
+import type {
+  GitlabInclude,
+  GitlabIncludeProject,
+  GitlabPipeline,
+} from '../gitlabci/types';
 import { replaceReferenceTags } from '../gitlabci/utils';
 import type { PackageDependency, PackageFile } from '../types';
+import {
+  filterIncludeFromGitlabPipeline,
+  isGitlabIncludeProject,
+  isNonEmptyObject,
+} from './common';
 
-function extractDepFromIncludeFile(includeObj: {
-  file: any;
-  project: string;
-  ref: string;
-}): PackageDependency {
+function extractDepFromIncludeFile(
+  includeObj: GitlabIncludeProject
+): PackageDependency {
   const dep: PackageDependency = {
     datasource: GitlabTagsDatasource.id,
     depName: includeObj.project,
@@ -25,28 +33,50 @@ function extractDepFromIncludeFile(includeObj: {
   return dep;
 }
 
+function getIncludeProjectsFromInclude(
+  includeValue: GitlabInclude[] | GitlabInclude
+): GitlabIncludeProject[] {
+  const includes = is.array(includeValue) ? includeValue : [includeValue];
+
+  // Filter out includes that dont have a file & project.
+  return includes.filter(isGitlabIncludeProject);
+}
+
+function getAllIncludeProjects(data: GitlabPipeline): GitlabIncludeProject[] {
+  // If Array, search each element.
+  if (is.array(data)) {
+    return (data as GitlabPipeline[])
+      .filter(isNonEmptyObject)
+      .map(getAllIncludeProjects)
+      .flat();
+  }
+
+  const childrenData = Object.values(filterIncludeFromGitlabPipeline(data))
+    .filter(isNonEmptyObject)
+    .map(getAllIncludeProjects)
+    .flat();
+
+  // Process include key.
+  if (data.include) {
+    childrenData.push(...getIncludeProjectsFromInclude(data.include));
+  }
+  return childrenData;
+}
+
 export function extractPackageFile(content: string): PackageFile | null {
   const deps: PackageDependency[] = [];
   const { platform, endpoint } = GlobalConfig.get();
   try {
-    // TODO: fix me (#9610)
-    const doc: any = load(replaceReferenceTags(content), {
+    const doc = load(replaceReferenceTags(content), {
       json: true,
-    });
-    let includes;
-    if (doc?.include && is.array(doc.include)) {
-      includes = doc.include;
-    } else {
-      includes = [doc.include];
-    }
+    }) as GitlabPipeline;
+    const includes = getAllIncludeProjects(doc);
     for (const includeObj of includes) {
-      if (includeObj?.file && includeObj.project) {
-        const dep = extractDepFromIncludeFile(includeObj);
-        if (platform === 'gitlab' && endpoint) {
-          dep.registryUrls = [endpoint.replace(regEx(/\/api\/v4\/?/), '')];
-        }
-        deps.push(dep);
+      const dep = extractDepFromIncludeFile(includeObj);
+      if (platform === 'gitlab' && endpoint) {
+        dep.registryUrls = [endpoint.replace(regEx(/\/api\/v4\/?/), '')];
       }
+      deps.push(dep);
     }
   } catch (err) /* istanbul ignore next */ {
     if (err.stack?.startsWith('YAMLException:')) {

lib/modules/manager/gitlabci/common.spec.ts

@@ -0,0 +1,16 @@
+import { isGitlabIncludeLocal } from './common';
+
+const includeLocal = { local: 'something' };
+const includeProject = { project: 'something' };
+
+describe('modules/manager/gitlabci/common', () => {
+  describe('isGitlabIncludeLocal()', () => {
+    it('returns true if GitlabInclude is GitlabIncludeLocal', () => {
+      expect(isGitlabIncludeLocal(includeLocal)).toBe(true);
+    });
+
+    it('returns false if GitlabInclude is not GitlabIncludeLocal', () => {
+      expect(isGitlabIncludeLocal(includeProject)).toBe(false);
+    });
+  });
+});

lib/modules/manager/gitlabci/common.ts

@@ -0,0 +1,8 @@
+import is from '@sindresorhus/is';
+import type { GitlabInclude, GitlabIncludeLocal } from '../gitlabci/types';
+
+export function isGitlabIncludeLocal(
+  include: GitlabInclude
+): include is GitlabIncludeLocal {
+  return !is.undefined((include as GitlabIncludeLocal).local);
+}

lib/modules/manager/gitlabci/extract.ts

@@ -4,6 +4,7 @@ import { logger } from '../../../logger';
 import { readLocalFile } from '../../../util/fs';
 import { trimLeadingSlash } from '../../../util/url';
 import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
+import { isGitlabIncludeLocal } from './common';
 import type { GitlabPipeline, Image, Job, Services } from './types';
 import { getGitlabDep, replaceReferenceTags } from './utils';
 
@@ -144,13 +145,11 @@ export async function extractAllPackageFiles(
     }
 
     if (is.array(doc?.include)) {
-      for (const includeObj of doc.include) {
-        if (is.string(includeObj.local)) {
-          const fileObj = trimLeadingSlash(includeObj.local);
-          if (!seen.has(fileObj)) {
-            seen.add(fileObj);
-            filesToExamine.push(fileObj);
-          }
+      for (const includeObj of doc.include.filter(isGitlabIncludeLocal)) {
+        const fileObj = trimLeadingSlash(includeObj.local);
+        if (!seen.has(fileObj)) {
+          seen.add(fileObj);
+          filesToExamine.push(fileObj);
         }
       }
     } else if (is.string(doc?.include)) {

lib/modules/manager/gitlabci/types.ts

@@ -1,9 +1,23 @@
-export interface GitlabInclude {
-  local?: string;
+export interface GitlabIncludeLocal {
+  local: string;
+}
+
+export interface GitlabIncludeProject {
+  project: string;
+  file?: string;
+  ref?: string;
+}
+
+export interface GitlabIncludeRemote {
+  remote: string;
+}
+
+export interface GitlabIncludeTemplate {
+  template: string;
 }
 
 export interface GitlabPipeline {
-  include?: GitlabInclude[] | string;
+  include?: GitlabInclude[] | GitlabInclude;
 }
 
 export interface ImageObject {
@@ -20,3 +34,9 @@ export interface Job {
 }
 export type Image = ImageObject | string;
 export type Services = (string | ServicesObject)[];
+export type GitlabInclude =
+  | GitlabIncludeLocal
+  | GitlabIncludeProject
+  | GitlabIncludeRemote
+  | GitlabIncludeTemplate
+  | string;