feat(manager/gradle): reimplement parser using tree-based approach (#…

…18663)
renovatebot · Nov 15, 2022 · b14336b · b14336b
1 parent 30eb3dd
commit b14336b
Show file tree

Hide file tree

Showing 8 changed files with 1,058 additions and 1,236 deletions.
diff --git a/lib/modules/manager/gradle/extract.spec.ts b/lib/modules/manager/gradle/extract.spec.ts
@@ -8,10 +8,16 @@ jest.mock('../../../util/fs');
 
 function mockFs(files: Record<string, string>): void {
   // TODO: fix types, jest is using wrong overload (#7154)
-  fs.readLocalFile.mockImplementation((fileName: string): Promise<any> => {
-    const content = files?.[fileName];
-    return Promise.resolve(content ?? '');
-  });
+  fs.getFileContentMap.mockImplementation(
+    (fileNames: string[]): Promise<any> => {
+      const fileContentMap: Record<string, string | null> = {};
+      for (const fileName of fileNames) {
+        fileContentMap[fileName] = files?.[fileName];
+      }
+
+      return Promise.resolve(fileContentMap);
+    }
+  );
 
   fs.getSiblingFileName.mockImplementation(
     (existingFileNameWithPath: string, otherFileName: string) => {
@@ -45,7 +51,9 @@ describe('modules/manager/gradle/extract', () => {
     const filename = 'build.gradle';
     const err = new Error('unknown');
 
-    jest.spyOn(parser, 'parseGradle').mockRejectedValueOnce(err);
+    jest.spyOn(parser, 'parseGradle').mockImplementationOnce(() => {
+      throw err;
+    });
     await extractAllPackageFiles({} as ExtractConfig, [filename]);
 
     expect(logger.logger.warn).toHaveBeenCalledWith(
@@ -641,7 +649,7 @@ describe('modules/manager/gradle/extract', () => {
     const res = await extractAllPackageFiles({} as ExtractConfig, [
       'gradleX/libs1.gradle',
       'gradle/libs2.gradle',
-      // 'gradle/libs3.gradle', is intentionally not listed here
+      'gradle/libs3.gradle',
       'gradleX/gradleX/libs4.gradle',
       'build.gradle',
       'gradle.properties',
@@ -663,6 +671,16 @@ describe('modules/manager/gradle/extract', () => {
           },
         ],
       },
+      {
+        packageFile: 'gradle/libs3.gradle',
+        deps: [
+          {
+            depName: 'com.google.guava:guava',
+            currentValue: '30.1-jre',
+            managerData: { packageFile: 'gradle/libs3.gradle' },
+          },
+        ],
+      },
       {
         packageFile: 'gradleX/libs1.gradle',
         deps: [
@@ -688,16 +706,6 @@ describe('modules/manager/gradle/extract', () => {
           },
         ],
       },
-      {
-        packageFile: 'gradle/libs3.gradle',
-        deps: [
-          {
-            depName: 'com.google.guava:guava',
-            currentValue: '30.1-jre',
-            managerData: { packageFile: 'gradle/libs3.gradle' },
-          },
-        ],
-      },
     ]);
   });
 

diff --git a/lib/modules/manager/gradle/extract.ts b/lib/modules/manager/gradle/extract.ts
@@ -1,6 +1,6 @@
 import upath from 'upath';
 import { logger } from '../../../logger';
-import { readLocalFile } from '../../../util/fs';
+import { getFileContentMap } from '../../../util/fs';
 import { MavenDatasource } from '../../datasource/maven';
 import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
 import { parseCatalog } from './extract/catalog';
@@ -44,6 +44,8 @@ export async function extractAllPackageFiles(
   const packageFilesByName: Record<string, PackageFile> = {};
   const registryUrls: string[] = [];
   const reorderedFiles = reorderFiles(packageFiles);
+  const fileContents = await getFileContentMap(packageFiles, true);
+
   for (const packageFile of reorderedFiles) {
     packageFilesByName[packageFile] = {
       packageFile,
@@ -53,7 +55,7 @@ export async function extractAllPackageFiles(
 
     try {
       // TODO #7154
-      const content = (await readLocalFile(packageFile, 'utf8'))!;
+      const content = fileContents[packageFile]!;
       const dir = upath.dirname(toAbsolutePath(packageFile));
 
       const updateVars = (newVars: PackageVariables): void => {
@@ -74,7 +76,7 @@ export async function extractAllPackageFiles(
           deps,
           urls,
           vars: gradleVars,
-        } = await parseGradle(content, vars, packageFile);
+        } = parseGradle(content, vars, packageFile, fileContents);
         urls.forEach((url) => {
           if (!registryUrls.includes(url)) {
             registryUrls.push(url);
@@ -101,6 +103,7 @@ export async function extractAllPackageFiles(
     // istanbul ignore else
     if (key) {
       let pkgFile = packageFilesByName[key];
+      // istanbul ignore if: won't happen if "apply from" processes only initially known files
       if (!pkgFile) {
         pkgFile = {
           packageFile: key,

diff --git a/lib/modules/manager/gradle/parser.spec.ts b/lib/modules/manager/gradle/parser.spec.ts
@@ -27,9 +27,9 @@ function mockFs(files: Record<string, string>): void {
 }
 
 describe('modules/manager/gradle/parser', () => {
-  it('handles end of input', async () => {
-    expect((await parseGradle('version = ')).deps).toBeEmpty();
-    expect((await parseGradle('id "foo.bar" version')).deps).toBeEmpty();
+  it('handles end of input', () => {
+    expect(parseGradle('version = ').deps).toBeEmpty();
+    expect(parseGradle('id "foo.bar" version').deps).toBeEmpty();
   });
 
   describe('variables', () => {
@@ -43,8 +43,8 @@ describe('modules/manager/gradle/parser', () => {
         ${'project.ext.foo.bar = "1.2.3"'} | ${'foo.bar'} | ${'1.2.3'}
         ${'rootProject.foobar = "1.2.3"'}  | ${'foobar'}  | ${'1.2.3'}
         ${'rootProject.foo.bar = "1.2.3"'} | ${'foo.bar'} | ${'1.2.3'}
-      `('$input', async ({ input, name, value }) => {
-        const { vars } = await parseGradle(input);
+      `('$input', ({ input, name, value }) => {
+        const { vars } = parseGradle(input);
         expect(vars).toContainKey(name);
         expect(vars[name]).toMatchObject({ key: name, value });
       });
@@ -55,8 +55,8 @@ describe('modules/manager/gradle/parser', () => {
         input                        | name     | value
         ${'set("foo", "1.2.3")'}     | ${'foo'} | ${'1.2.3'}
         ${'version("foo", "1.2.3")'} | ${'foo'} | ${'1.2.3'}
-      `('$input', async ({ input, name, value }) => {
-        const { vars } = await parseGradle(input);
+      `('$input', ({ input, name, value }) => {
+        const { vars } = parseGradle(input);
         expect(vars).toContainKey(name);
         expect(vars[name]).toMatchObject({ key: name, value });
       });
@@ -70,8 +70,8 @@ describe('modules/manager/gradle/parser', () => {
         ${'"foo:bar:1.2.3"'}           | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
         ${'"foo:bar:1.2.3@zip"'}       | ${{ depName: 'foo:bar', currentValue: '1.2.3', dataType: 'zip' }}
         ${'foo.bar = "foo:bar:1.2.3"'} | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
-      `('$input', async ({ input, output }) => {
-        const { deps } = await parseGradle(input);
+      `('$input', ({ input, output }) => {
+        const { deps } = parseGradle(input);
         expect(deps).toMatchObject([output].filter(Boolean));
       });
     });
@@ -87,8 +87,8 @@ describe('modules/manager/gradle/parser', () => {
         ${'foo = "1.2.3"'}                   | ${'"foo:bar_$foo:4.5.6"'}     | ${{ depName: 'foo:bar_1.2.3', managerData: { fileReplacePosition: 28 } }}
         ${'baz = "1.2.3"'}                   | ${'foobar = "foo:bar:$baz"'}  | ${{ depName: 'foo:bar', currentValue: '1.2.3', groupName: 'baz' }}
         ${'foo = "${bar}"; baz = "1.2.3"'}   | ${'"foo:bar:${baz}"'}         | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
-      `('$def | $str', async ({ def, str, output }) => {
-        const { deps } = await parseGradle([def, str].join('\n'));
+      `('$def | $str', ({ def, str, output }) => {
+        const { deps } = parseGradle([def, str].join('\n'));
         expect(deps).toMatchObject([output].filter(Boolean));
       });
     });
@@ -98,13 +98,15 @@ describe('modules/manager/gradle/parser', () => {
         def                | str                                                                               | output
         ${''}              | ${'group: "foo", name: "bar", version: "1.2.3"'}                                  | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
         ${''}              | ${'group: "foo", name: "bar", version: baz'}                                      | ${null}
+        ${''}              | ${'group: "foo", name: "bar", version: "1.2.3@@@"'}                               | ${null}
         ${'baz = "1.2.3"'} | ${'group: "foo", name: "bar", version: baz'}                                      | ${{ depName: 'foo:bar', currentValue: '1.2.3', groupName: 'baz' }}
+        ${'baz = "1.2.3"'} | ${'group: "foo", name: "bar", version: "${baz}456"'}                              | ${{ depName: 'foo:bar', skipReason: 'unknown-version' }}
         ${''}              | ${'(group: "foo", name: "bar", version: "1.2.3", classifier: "sources")'}         | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
         ${''}              | ${'(group: "foo", name: "bar", version: "1.2.3") {exclude module: "spring-jcl"}'} | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
         ${''}              | ${"implementation platform(group: 'foo', name: 'bar', version: '1.2.3')"}         | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
         ${''}              | ${'(group = "foo", name = "bar", version = "1.2.3")'}                             | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
-      `('$def | $str', async ({ def, str, output }) => {
-        const { deps } = await parseGradle([def, str].join('\n'));
+      `('$def | $str', ({ def, str, output }) => {
+        const { deps } = parseGradle([def, str].join('\n'));
         expect(deps).toMatchObject([output].filter(Boolean));
       });
     });
@@ -127,8 +129,8 @@ describe('modules/manager/gradle/parser', () => {
         ${'baz = "1.2.3"'}  | ${'id("foo.bar") version baz'}        | ${{ depName: 'foo.bar', packageName: 'foo.bar:foo.bar.gradle.plugin', currentValue: '1.2.3' }}
         ${''}               | ${'kotlin("jvm") version "1.3.71"'}   | ${{ depName: 'org.jetbrains.kotlin.jvm', packageName: 'org.jetbrains.kotlin.jvm:org.jetbrains.kotlin.jvm.gradle.plugin', currentValue: '1.3.71' }}
         ${'baz = "1.3.71"'} | ${'kotlin("jvm") version baz'}        | ${{ depName: 'org.jetbrains.kotlin.jvm', packageName: 'org.jetbrains.kotlin.jvm:org.jetbrains.kotlin.jvm.gradle.plugin', currentValue: '1.3.71' }}
-      `('$def | $input', async ({ def, input, output }) => {
-        const { deps } = await parseGradle([def, input].join('\n'));
+      `('$def | $input', ({ def, input, output }) => {
+        const { deps } = parseGradle([def, input].join('\n'));
         expect(deps).toMatchObject([output].filter(Boolean));
       });
     });
@@ -143,8 +145,8 @@ describe('modules/manager/gradle/parser', () => {
         ${'google { content { includeGroup "foo" } }'} | ${GOOGLE_REPO}
         ${'gradlePluginPortal()'}                      | ${GRADLE_PLUGIN_PORTAL_REPO}
         ${'jcenter()'}                                 | ${JCENTER_REPO}
-      `('$input', async ({ input, output }) => {
-        const { urls } = await parseGradle(input);
+      `('$input', ({ input, output }) => {
+        const { urls } = parseGradle(input);
         expect(urls).toStrictEqual([output].filter(Boolean));
       });
     });
@@ -166,9 +168,9 @@ describe('modules/manager/gradle/parser', () => {
         ${''}                       | ${'maven { url = uri("https://foo.bar/baz") }'}      | ${'https://foo.bar/baz'}
         ${'base="https://foo.bar"'} | ${'maven { url = uri("${base}/baz") }'}              | ${'https://foo.bar/baz'}
         ${'base="https://foo.bar"'} | ${'maven { name = "baz"\nurl = "${base}/${name}" }'} | ${'https://foo.bar/baz'}
-      `('$def | $input', async ({ def, input, url }) => {
+      `('$def | $input', ({ def, input, url }) => {
         const expected = [url].filter(Boolean);
-        const { urls } = await parseGradle([def, input].join('\n'));
+        const { urls } = parseGradle([def, input].join('\n'));
         expect(urls).toStrictEqual(expected);
       });
     });
@@ -183,11 +185,12 @@ describe('modules/manager/gradle/parser', () => {
       ${''}                                         | ${'library(["foo.bar", "foo", "bar"]).version("1.2.3")'}        | ${null}
       ${''}                                         | ${'library("foo", "bar", "baz", "qux"]).version("1.2.3")'}      | ${null}
       ${''}                                         | ${'library("foo.bar", "foo", "bar").version("1.2.3", "4.5.6")'} | ${null}
+      ${''}                                         | ${'library("foo", bar, "baz").version("1.2.3")'}                | ${null}
       ${'group = "foo"; artifact="bar"'}            | ${'library("foo.bar", group, artifact).version("1.2.3")'}       | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
       ${'library("foo-bar_baz-qux", "foo", "bar")'} | ${'"${libs.foo.bar.baz.qux}:1.2.3"'}                            | ${{ depName: 'foo:bar', currentValue: '1.2.3' }}
-    `('$def | $str', async ({ def, str, output }) => {
+    `('$def | $str', ({ def, str, output }) => {
       const input = [def, str].join('\n');
-      const { deps } = await parseGradle(input);
+      const { deps } = parseGradle(input);
       expect(deps).toMatchObject([output].filter(Boolean));
     });
   });
@@ -209,16 +212,16 @@ describe('modules/manager/gradle/parser', () => {
       ${'mutableSetOf("foo", "bar", "baz")'}                       | ${{ depName: 'foo:bar', currentValue: 'baz', skipReason: 'ignored' }}
       ${'stages("foo", "bar", "baz")'}                             | ${{ depName: 'foo:bar', currentValue: 'baz', skipReason: 'ignored' }}
       ${'mapScalar("foo", "bar", "baz")'}                          | ${{ depName: 'foo:bar', currentValue: 'baz', skipReason: 'ignored' }}
-    `('$input', async ({ input, output }) => {
-      const { deps } = await parseGradle(input);
+    `('$input', ({ input, output }) => {
+      const { deps } = parseGradle(input);
       expect(deps).toMatchObject([output].filter(Boolean));
     });
   });
 
   describe('calculations', () => {
-    it('calculates offset', async () => {
+    it('calculates offset', () => {
       const content = "'foo:bar:1.2.3'";
-      const { deps } = await parseGradle(content);
+      const { deps } = parseGradle(content);
       const [res] = deps;
       const idx = content
         // TODO #7154
@@ -227,9 +230,9 @@ describe('modules/manager/gradle/parser', () => {
       expect(idx).toBe(0);
     });
 
-    it('parses fixture from "gradle" manager', async () => {
+    it('parses fixture from "gradle" manager', () => {
       const content = Fixtures.get('build.gradle.example1');
-      const { deps } = await parseGradle(content, {}, 'build.gradle');
+      const { deps } = parseGradle(content, {}, 'build.gradle');
       const replacementIndices = deps.map(({ managerData, currentValue }) =>
         // TODO #7154
         content.slice(managerData!.fileReplacePosition).indexOf(currentValue!)
@@ -331,16 +334,22 @@ describe('modules/manager/gradle/parser', () => {
       ${''}                  | ${'apply(from = File("foo", "bar.gradle"))'}        | ${validOutput}
       ${'base="foo"'}        | ${'apply(from = File(base, "bar.gradle"))'}         | ${validOutput}
       ${'base="foo"'}        | ${'apply(from = File("${base}", "bar.gradle"))'}    | ${validOutput}
-    `('$def | $input', async ({ def, input, output }) => {
-      const { vars } = await parseGradle([def, input].join('\n'));
+    `('$def | $input', ({ def, input, output }) => {
+      const { vars } = parseGradle(
+        [def, input].join('\n'),
+        {},
+        '',
+        fileContents
+      );
       expect(vars).toMatchObject(output);
     });
 
-    it('recursion check', async () => {
-      const { vars } = await parseGradle(
+    it('recursion check', () => {
+      const { vars } = parseGradle(
         'apply from: "foo/bar.gradle"',
         {},
         '',
+        fileContents,
         3
       );
       expect(logger.logger.debug).toHaveBeenCalledWith(