feat(batect): Automatically extract dependencies for files included i…

…nto Batect configuration (#8091)

* Automatically extract dependencies for files included into Batect configuration.

* Fix issue running tests on Windows.

* Simplify language in readme.

* Use sets rather than arrays to manage backlog of files to examine.

* Remove explicitly setting manager name.

* Address PR feedback.

Co-authored-by: Jamie Magee <JamieMagee@users.noreply.github.com>
Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

lib/manager/batect/__fixtures__/invalid/batect.yml

@@ -0,0 +1 @@
+nothing here

lib/manager/batect/__fixtures__/no-containers-or-includes/batect.yml

@@ -0,0 +1 @@
+something_else: some_value

lib/manager/batect/__fixtures__/valid/another-include.yml

@@ -0,0 +1,8 @@
+include:
+  - type: git
+    repo: https://another-include.com/my-repo.git
+    ref: 4.5.6
+
+containers:
+  another-include-container-1:
+    image: ubuntu:19.10

lib/manager/batect/__fixtures__/batect.yml → lib/manager/batect/__fixtures__/valid/batect.yml

@@ -15,10 +15,11 @@ containers:
     image: postgres:9.6.20@sha256:166179811e4c75f8a092367afed6091208c8ecf60b111c7e49f29af45ca05e08
 
 include:
-  - some/file/include.yml
+  - include.yml
+  - subdir/file.yml
 
   - type: file
-    path: a/valid/file/include.yml
+    path: another-include.yml
 
   - type: file
     repo: https://file.includes/should/not/have/repo.git

lib/manager/batect/__fixtures__/valid/include.yml

@@ -0,0 +1,11 @@
+include:
+  - another-include.yml
+
+  - type: git
+    repo: https://include.com/my-repo.git
+    ref: 4.5.6
+
+containers:
+  include-container-1:
+    image: ubuntu:20.10
+

lib/manager/batect/__fixtures__/valid/subdir/file.yml

@@ -0,0 +1,11 @@
+include:
+  - ../include.yml
+
+  - type: git
+    repo: https://file.com/my-repo.git
+    ref: 4.5.6
+
+containers:
+  file-container-1:
+    image: ubuntu:19.04
+

lib/manager/batect/extract.spec.ts

@@ -1,15 +1,11 @@
-import { readFileSync } from 'fs';
 import { id as gitTagDatasource } from '../../datasource/git-tags';
 import { id as dockerVersioning } from '../../versioning/docker';
 import { id as semverVersioning } from '../../versioning/semver';
 import { PackageDependency } from '../common';
 import { getDep } from '../dockerfile/extract';
-import { extractPackageFile } from './extract';
+import { extractAllPackageFiles } from './extract';
 
-const sampleFile = readFileSync(
-  'lib/manager/batect/__fixtures__/batect.yml',
-  'utf8'
-);
+const fixturesDir = 'lib/manager/batect/__fixtures__';
 
 function createDockerDependency(tag: string): PackageDependency {
   return {
@@ -30,34 +26,79 @@ function createGitDependency(repo: string, version: string): PackageDependency {
 
 describe('lib/manager/batect/extract', () => {
   describe('extractPackageFile()', () => {
-    it('returns null for empty configuration file', () => {
-      expect(extractPackageFile('')).toBeNull();
+    it('returns empty array for empty configuration file', async () => {
+      expect(
+        await extractAllPackageFiles({}, [`${fixturesDir}/empty/batect.yml`])
+      ).toEqual([]);
     });
 
-    it('returns null for non-object configuration file', () => {
-      expect(extractPackageFile('nothing here')).toBeNull();
+    it('returns empty array for non-object configuration file', async () => {
+      expect(
+        await extractAllPackageFiles({}, [`${fixturesDir}/invalid/batect.yml`])
+      ).toEqual([]);
     });
 
-    it('returns null for malformed configuration file', () => {
-      expect(extractPackageFile('nothing here\n:::::::')).toBeNull();
-    });
+    it('returns an a package file with no dependencies for configuration file without containers or includes', async () => {
+      const result = await extractAllPackageFiles({}, [
+        `${fixturesDir}/no-containers-or-includes/batect.yml`,
+      ]);
 
-    it('returns null for configuration file without containers', () => {
-      expect(extractPackageFile('something_else: some_value')).toBeNull();
+      expect(result).toEqual([
+        {
+          packageFile: `${fixturesDir}/no-containers-or-includes/batect.yml`,
+          deps: [],
+        },
+      ]);
     });
 
-    it('extracts all available images from a valid Batect configuration file', () => {
-      const res = extractPackageFile(sampleFile);
+    it('extracts all available images and bundles from a valid Batect configuration file, including dependencies in included files', async () => {
+      const result = await extractAllPackageFiles({}, [
+        `${fixturesDir}/valid/batect.yml`,
+      ]);
 
-      expect(res.deps).toEqual([
-        createDockerDependency('alpine:1.2.3'),
-        createDockerDependency('alpine:1.2.3'),
-        createDockerDependency('ubuntu:20.04'),
-        createDockerDependency(
-          'postgres:9.6.20@sha256:166179811e4c75f8a092367afed6091208c8ecf60b111c7e49f29af45ca05e08'
-        ),
-        createGitDependency('https://includes.com/my-repo.git', '1.2.3'),
-        createGitDependency('https://includes.com/my-other-repo.git', '4.5.6'),
+      expect(
+        result.sort((a, b) => a.packageFile.localeCompare(b.packageFile))
+      ).toEqual([
+        {
+          packageFile: `${fixturesDir}/valid/another-include.yml`,
+          deps: [
+            createDockerDependency('ubuntu:19.10'),
+            createGitDependency(
+              'https://another-include.com/my-repo.git',
+              '4.5.6'
+            ),
+          ],
+        },
+        {
+          packageFile: `${fixturesDir}/valid/batect.yml`,
+          deps: [
+            createDockerDependency('alpine:1.2.3'),
+            createDockerDependency('alpine:1.2.3'),
+            createDockerDependency('ubuntu:20.04'),
+            createDockerDependency(
+              'postgres:9.6.20@sha256:166179811e4c75f8a092367afed6091208c8ecf60b111c7e49f29af45ca05e08'
+            ),
+            createGitDependency('https://includes.com/my-repo.git', '1.2.3'),
+            createGitDependency(
+              'https://includes.com/my-other-repo.git',
+              '4.5.6'
+            ),
+          ],
+        },
+        {
+          packageFile: `${fixturesDir}/valid/include.yml`,
+          deps: [
+            createDockerDependency('ubuntu:20.10'),
+            createGitDependency('https://include.com/my-repo.git', '4.5.6'),
+          ],
+        },
+        {
+          packageFile: `${fixturesDir}/valid/subdir/file.yml`,
+          deps: [
+            createDockerDependency('ubuntu:19.04'),
+            createGitDependency('https://file.com/my-repo.git', '4.5.6'),
+          ],
+        },
       ]);
     });
   });

lib/manager/batect/extract.ts

@@ -1,12 +1,19 @@
 import { safeLoad } from 'js-yaml';
+import * as upath from 'upath';
 
 import { id as gitTagDatasource } from '../../datasource/git-tags';
 import { logger } from '../../logger';
+import { readLocalFile } from '../../util/fs';
 import { id as dockerVersioning } from '../../versioning/docker';
 import { id as semverVersioning } from '../../versioning/semver';
-import { PackageDependency, PackageFile } from '../common';
+import { ExtractConfig, PackageDependency, PackageFile } from '../common';
 import { getDep } from '../dockerfile/extract';
-import { BatectConfig, BatectGitInclude } from './types';
+import {
+  BatectConfig,
+  BatectFileInclude,
+  BatectGitInclude,
+  BatectInclude,
+} from './types';
 
 function loadConfig(content: string): BatectConfig {
   const config = safeLoad(content);
@@ -46,15 +53,18 @@ function extractImageDependencies(config: BatectConfig): PackageDependency[] {
   return deps;
 }
 
+function includeIsGitInclude(
+  include: BatectInclude
+): include is BatectGitInclude {
+  return typeof include === 'object' && include.type === 'git';
+}
+
 function extractGitBundles(config: BatectConfig): BatectGitInclude[] {
   if (config.include === undefined) {
     return [];
   }
 
-  return config.include.filter(
-    (include): include is BatectGitInclude =>
-      typeof include === 'object' && include.type === 'git'
-  );
+  return config.include.filter(includeIsGitInclude);
 }
 
 function createBundleDependency(bundle: BatectGitInclude): PackageDependency {
@@ -76,10 +86,45 @@ function extractBundleDependencies(config: BatectConfig): PackageDependency[] {
   return deps;
 }
 
-export function extractPackageFile(
+function includeIsStringFileInclude(include: BatectInclude): include is string {
+  return typeof include === 'string';
+}
+
+function includeIsObjectFileInclude(
+  include: BatectInclude
+): include is BatectFileInclude {
+  return typeof include === 'object' && include.type === 'file';
+}
+
+function extractReferencedConfigFiles(
+  config: BatectConfig,
+  fileName: string
+): string[] {
+  if (config.include === undefined) {
+    return [];
+  }
+
+  const dirName = upath.dirname(fileName);
+
+  const paths = [
+    ...config.include.filter(includeIsStringFileInclude),
+    ...config.include
+      .filter(includeIsObjectFileInclude)
+      .map((include) => include.path),
+  ].filter((p) => p !== undefined && p !== null);
+
+  return paths.map((p) => upath.join(dirName, p));
+}
+
+interface ExtractionResult {
+  deps: PackageDependency[];
+  referencedConfigFiles: string[];
+}
+
+function extractPackageFile(
   content: string,
-  fileName?: string
-): PackageFile | null {
+  fileName: string
+): ExtractionResult | null {
   logger.debug({ fileName }, 'batect.extractPackageFile()');
 
   try {
@@ -89,11 +134,12 @@ export function extractPackageFile(
       ...extractBundleDependencies(config),
     ];
 
-    if (deps.length === 0) {
-      return null;
-    }
+    const referencedConfigFiles = extractReferencedConfigFiles(
+      config,
+      fileName
+    );
 
-    return { deps };
+    return { deps, referencedConfigFiles };
   } catch (err) {
     logger.warn(
       { err, fileName },
@@ -103,3 +149,36 @@ export function extractPackageFile(
     return null;
   }
 }
+
+export async function extractAllPackageFiles(
+  config: ExtractConfig,
+  packageFiles: string[]
+): Promise<PackageFile[] | null> {
+  const filesToExamine = new Set<string>(packageFiles);
+  const filesAlreadyExamined = new Set<string>();
+  const results: PackageFile[] = [];
+
+  while (filesToExamine.size > 0) {
+    const packageFile = filesToExamine.values().next().value;
+    filesToExamine.delete(packageFile);
+    filesAlreadyExamined.add(packageFile);
+
+    const content = await readLocalFile(packageFile, 'utf8');
+    const result = extractPackageFile(content, packageFile);
+
+    if (result !== null) {
+      result.referencedConfigFiles.forEach((f) => {
+        if (!filesAlreadyExamined.has(f) && !filesToExamine.has(f)) {
+          filesToExamine.add(f);
+        }
+      });
+
+      results.push({
+        packageFile,
+        deps: result.deps,
+      });
+    }
+  }
+
+  return results;
+}

lib/manager/batect/index.ts

@@ -1,6 +1,6 @@
-import { extractPackageFile } from './extract';
+import { extractAllPackageFiles } from './extract';
 
-export { extractPackageFile };
+export { extractAllPackageFiles };
 
 export const defaultConfig = {
   fileMatch: ['(^|/)batect(-bundle)?\\.yml$'],

lib/manager/batect/readme.md

@@ -7,7 +7,7 @@ For updates to Batect itself, see [batect-wrapper](../batect-wrapper).
 By default, the manager searches for files called `batect.yml` or `batect-bundle.yml`.
 
 If you keep your Batect configuration in other files, you'll need to tell Renovate where to find them.
-This includes files included into your main configuration file with `include`.
+Files included in your main configuration file with `include` don't need to be listed.
 
 You do this by creating a `"batect"` object in your `renovate.json` file.
 This object should contain a `fileMatch` array with regular expressions that match the configuration file names.

lib/manager/batect/types.ts

@@ -11,6 +11,7 @@ export type BatectInclude = string | BatectFileInclude | BatectGitInclude;
 
 export interface BatectFileInclude {
   type: 'file';
+  path: string;
 }
 
 export interface BatectGitInclude {