diff --git a/.gitignore b/.gitignore index d536d0bb5ea..1f7c828e001 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ config.local.json /node_modules/ local.log /patches/ -/dist +**/dist tmp .DS_Store /package-lock.json @@ -24,3 +24,4 @@ report.[0-9]*.[0-9]*.[0-9]*.[0-9]*.json /test/acceptance/workspaces/**/project/ /test/acceptance/workspaces/**/target/ test/acceptance/workspaces/**/.gradle +test/**/.gradle diff --git a/package.json b/package.json index d1833088e17..7b66d786651 100644 --- a/package.json +++ b/package.json @@ -53,7 +53,7 @@ "author": "snyk.io", "license": "Apache-2.0", "dependencies": { - "@snyk/cli-interface": "2.8.1", + "@snyk/cli-interface": "2.9.0", "@snyk/dep-graph": "1.19.3", "@snyk/gemfile": "1.2.0", "@snyk/graphlib": "2.1.9-patch", diff --git a/src/lib/find-files.ts b/src/lib/find-files.ts index c89eb3b64a7..bc9708141dc 100644 --- a/src/lib/find-files.ts +++ b/src/lib/find-files.ts @@ -39,6 +39,11 @@ export async function getStats(path: string): Promise { }); } +interface FindFilesRes { + files: string[]; + allFilesFound: string[]; +} + /** * Find all files in given search path. Returns paths to files found. * @@ -52,11 +57,13 @@ export async function find( ignore: string[] = [], filter: string[] = [], levelsDeep = 4, -): Promise { +): Promise { const found: string[] = []; + const foundAll: string[] = []; + // ensure we ignore find against node_modules path. if (path.endsWith('node_modules')) { - return found; + return { files: found, allFilesFound: foundAll }; } // ensure node_modules is always ignored if (!ignore.includes('node_modules')) { @@ -64,22 +71,36 @@ export async function find( } try { if (levelsDeep < 0) { - return found; + return { files: found, allFilesFound: foundAll }; } else { levelsDeep--; } const fileStats = await getStats(path); if (fileStats.isDirectory()) { - const files = await findInDirectory(path, ignore, filter, levelsDeep); + const { files, allFilesFound } = await findInDirectory( + path, + ignore, + filter, + levelsDeep, + ); found.push(...files); + foundAll.push(...allFilesFound); } else if (fileStats.isFile()) { const fileFound = findFile(path, filter); if (fileFound) { found.push(fileFound); + foundAll.push(fileFound); } } - - return filterForDefaultManifests(found); + const filteredOutFiles = foundAll.filter((f) => !found.includes(f)); + if (filteredOutFiles.length) { + debug( + `Filtered out ${filteredOutFiles.length}/${ + foundAll.length + } files: ${foundAll.join(', ')}`, + ); + } + return { files: filterForDefaultManifests(found), allFilesFound: foundAll }; } catch (err) { throw new Error(`Error finding files in path '${path}'.\n${err.message}`); } @@ -102,7 +123,7 @@ async function findInDirectory( ignore: string[] = [], filter: string[] = [], levelsDeep = 4, -): Promise { +): Promise { const files = await readDirectory(path); const toFind = files .filter((file) => !ignore.includes(file)) @@ -110,12 +131,22 @@ async function findInDirectory( const resolvedPath = pathLib.resolve(path, file); if (!fs.existsSync(resolvedPath)) { debug('File does not seem to exist, skipping: ', file); - return []; + return { files: [], allFilesFound: [] }; } return find(resolvedPath, ignore, filter, levelsDeep); }); + const found = await Promise.all(toFind); - return Array.prototype.concat.apply([], found); + return { + files: Array.prototype.concat.apply( + [], + found.map((f) => f.files), + ), + allFilesFound: Array.prototype.concat.apply( + [], + found.map((f) => f.allFilesFound), + ), + }; } function filterForDefaultManifests(files: string[]): string[] { diff --git a/src/lib/plugins/get-deps-from-plugin.ts b/src/lib/plugins/get-deps-from-plugin.ts index 4fd4117e65f..79e91696e06 100644 --- a/src/lib/plugins/get-deps-from-plugin.ts +++ b/src/lib/plugins/get-deps-from-plugin.ts @@ -1,4 +1,6 @@ import * as debugModule from 'debug'; +import * as pathLib from 'path'; +import chalk from 'chalk'; import { legacyPlugin as pluginApi } from '@snyk/cli-interface'; import { find } from '../find-files'; import { Options, TestOptions, MonitorOptions } from '../types'; @@ -17,6 +19,7 @@ import analytics = require('../analytics'); import { convertSingleResultToMultiCustom } from './convert-single-splugin-res-to-multi-custom'; import { convertMultiResultToMultiCustom } from './convert-multi-plugin-res-to-multi-custom'; import { processYarnWorkspaces } from './nodejs-plugin/yarn-workspaces-parser'; +import { ScannedProject } from '@snyk/cli-interface/legacy/common'; const debug = debugModule('snyk-test'); @@ -42,7 +45,7 @@ export async function getDepsFromPlugin( const scanType = options.yarnWorkspaces ? 'yarnWorkspaces' : 'allProjects'; const levelsDeep = options.detectionDepth; const ignore = options.exclude ? options.exclude.split(',') : []; - const targetFiles = await find( + const { files: targetFiles, allFilesFound } = await find( root, ignore, multiProjectProcessors[scanType].files, @@ -62,8 +65,9 @@ export async function getDepsFromPlugin( options, targetFiles, ); + const scannedProjects = inspectRes.scannedProjects; const analyticData = { - scannedProjects: inspectRes.scannedProjects.length, + scannedProjects: scannedProjects.length, targetFiles, packageManagers: targetFiles.map((file) => detectPackageManagerFromFile(file), @@ -72,6 +76,18 @@ export async function getDepsFromPlugin( ignore, }; analytics.add(scanType, analyticData); + debug( + `Found ${scannedProjects.length} projects from ${allFilesFound.length} detected manifests`, + ); + const userWarningMessage = warnSomeGradleManifestsNotScanned( + scannedProjects, + allFilesFound, + root, + ); + + if (!options.json && userWarningMessage) { + console.warn(chalk.bold.red(userWarningMessage)); + } return inspectRes; } @@ -105,3 +121,35 @@ export async function getDepsFromPlugin( ); return convertMultiResultToMultiCustom(inspectRes, options.packageManager); } + +export function warnSomeGradleManifestsNotScanned( + scannedProjects: ScannedProject[], + allFilesFound: string[], + root: string, +): string | null { + const gradleTargetFilesFilter = (targetFile) => + targetFile && + (targetFile.endsWith('build.gradle') || + targetFile.endsWith('build.gradle.kts')); + const scannedGradleFiles = scannedProjects + .map((p) => { + const targetFile = p.meta?.targetFile || p.targetFile; + return targetFile ? pathLib.resolve(root, targetFile) : null; + }) + .filter(gradleTargetFilesFilter); + const detectedGradleFiles = allFilesFound.filter(gradleTargetFilesFilter); + const diff = detectedGradleFiles.filter( + (file) => !scannedGradleFiles.includes(file), + ); + + debug( + `These Gradle manifests did not return any dependency results:\n${diff.join( + ',\n', + )}`, + ); + + if (diff.length > 0) { + return `✗ ${diff.length}/${detectedGradleFiles.length} detected Gradle manifests did not return dependencies.\nThey may have errored or were not included as part of a multi-project build. You may need to scan them individually with --file=path/to/file. Run with \`-d\` for more info.`; + } + return null; +} diff --git a/src/lib/plugins/get-extra-project-count.ts b/src/lib/plugins/get-extra-project-count.ts index 8c40a71d7d0..8d31f99a6c7 100644 --- a/src/lib/plugins/get-extra-project-count.ts +++ b/src/lib/plugins/get-extra-project-count.ts @@ -19,7 +19,11 @@ export async function getExtraProjectCount( return inspectResult.plugin.meta.allSubProjectNames.length; } try { - const extraTargetFiles = await find(root, [], AUTO_DETECTABLE_FILES); + const { files: extraTargetFiles } = await find( + root, + [], + AUTO_DETECTABLE_FILES, + ); const foundProjectsCount = extraTargetFiles.length > 1 ? extraTargetFiles.length - 1 : undefined; return foundProjectsCount; diff --git a/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts b/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts index 0d2a9f13f33..71e864ec77b 100644 --- a/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts +++ b/src/lib/plugins/nodejs-plugin/npm-lock-parser.ts @@ -58,6 +58,6 @@ export async function parse( strictOutOfSync, ); } finally { - await spinner.clear(resolveModuleSpinnerLabel); + await spinner.clear(resolveModuleSpinnerLabel)(); } } diff --git a/test/acceptance/cli-test/cli-test.all-projects.spec.ts b/test/acceptance/cli-test/cli-test.all-projects.spec.ts index 90862be8034..2e9a07a69f0 100644 --- a/test/acceptance/cli-test/cli-test.all-projects.spec.ts +++ b/test/acceptance/cli-test/cli-test.all-projects.spec.ts @@ -4,40 +4,112 @@ import * as depGraphLib from '@snyk/dep-graph'; import { CommandResult } from '../../../src/cli/commands/types'; import { AcceptanceTests } from './cli-test.acceptance.test'; import { getWorkspaceJSON } from '../workspace-helper'; +import * as getDepsFromPlugin from '../../../src/lib/plugins/get-deps-from-plugin'; + +const simpleGradleGraph = depGraphLib.createFromJSON({ + schemaVersion: '1.2.0', + pkgManager: { + name: 'gradle', + }, + pkgs: [ + { + id: 'gradle-monorepo@0.0.0', + info: { + name: 'gradle-monorepo', + version: '0.0.0', + }, + }, + ], + graph: { + rootNodeId: 'root-node', + nodes: [ + { + nodeId: 'root-node', + pkgId: 'gradle-monorepo@0.0.0', + deps: [], + }, + ], + }, +}); export const AllProjectsTests: AcceptanceTests = { language: 'Mixed', tests: { - '`test kotlin-monorepo --all-projects` scans kotlin files': ( + '`test gradle-with-orphaned-build-file --all-projects` warns user': ( params, utils, ) => async (t) => { utils.chdirWorkspaces(); - const simpleGradleGraph = depGraphLib.createFromJSON({ - schemaVersion: '1.2.0', - pkgManager: { - name: 'gradle', - }, - pkgs: [ - { - id: 'gradle-monorepo@0.0.0', - info: { - name: 'gradle-monorepo', - version: '0.0.0', - }, - }, - ], - graph: { - rootNodeId: 'root-node', - nodes: [ - { - nodeId: 'root-node', - pkgId: 'gradle-monorepo@0.0.0', - deps: [], + const plugin = { + async inspect() { + return { + plugin: { + name: 'bundled:gradle', + runtime: 'unknown', + meta: {}, }, - ], + scannedProjects: [ + { + meta: { + gradleProjectName: 'root-proj', + versionBuildInfo: { + gradleVersion: '6.5', + }, + targetFile: 'build.gradle', + }, + depGraph: simpleGradleGraph, + }, + { + meta: { + gradleProjectName: 'root-proj/subproj', + versionBuildInfo: { + gradleVersion: '6.5', + }, + targetFile: 'subproj/build.gradle', + }, + depGraph: simpleGradleGraph, + }, + ], + }; }, - }); + }; + const loadPlugin = sinon.stub(params.plugins, 'loadPlugin'); + t.teardown(loadPlugin.restore); + loadPlugin.withArgs('gradle').returns(plugin); + loadPlugin.callThrough(); + // read data from console.log + let stdoutMessages = ''; + const stubConsoleLog = (msg: string) => (stdoutMessages += msg); + const stubbedConsole = sinon + .stub(console, 'warn') + .callsFake(stubConsoleLog); + const result: CommandResult = await params.cli.test( + 'gradle-with-orphaned-build-file', + { + allProjects: true, + detectionDepth: 3, + }, + ); + t.same( + stdoutMessages, + '✗ 1/3 detected Gradle manifests did not return dependencies.\n' + + 'They may have errored or were not included as part of a multi-project build. You may need to scan them individually with --file=path/to/file. Run with `-d` for more info.', + ); + stubbedConsole.restore(); + t.ok(stubbedConsole.calledOnce); + t.ok(loadPlugin.withArgs('gradle').calledOnce, 'calls gradle plugin'); + + t.match( + result.getDisplayResults(), + 'Tested 2 projects', + 'Detected 2 projects', + ); + }, + '`test kotlin-monorepo --all-projects` scans kotlin files': ( + params, + utils, + ) => async (t) => { + utils.chdirWorkspaces(); const plugin = { async inspect() { return { diff --git a/test/acceptance/workspaces/gradle-with-orphaned-build-file/build.gradle b/test/acceptance/workspaces/gradle-with-orphaned-build-file/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/acceptance/workspaces/gradle-with-orphaned-build-file/orphaned/build.gradle b/test/acceptance/workspaces/gradle-with-orphaned-build-file/orphaned/build.gradle new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/acceptance/workspaces/gradle-with-orphaned-build-file/settings.gradle b/test/acceptance/workspaces/gradle-with-orphaned-build-file/settings.gradle new file mode 100644 index 00000000000..90289e8a32b --- /dev/null +++ b/test/acceptance/workspaces/gradle-with-orphaned-build-file/settings.gradle @@ -0,0 +1,5 @@ +rootProject.name = 'root-proj' + +include 'subproj' + + diff --git a/test/acceptance/workspaces/gradle-with-orphaned-build-file/subproj/build.gradle b/test/acceptance/workspaces/gradle-with-orphaned-build-file/subproj/build.gradle new file mode 100644 index 00000000000..301e7be03bc --- /dev/null +++ b/test/acceptance/workspaces/gradle-with-orphaned-build-file/subproj/build.gradle @@ -0,0 +1,51 @@ +apply plugin: 'java' +apply plugin: 'maven' + +group = 'com.github.jitpack' + +sourceCompatibility = 1.8 // java 8 +targetCompatibility = 1.8 + +repositories { + mavenCentral() +} + +dependencies { + // Gradle 3+ will not pick up "compile" dependencies for "compileOnly" + // Gradle 2 will, so for configuration-matching tests we use "runtime" + runtime 'com.google.guava:guava:18.0' + runtime 'batik:batik-dom:1.6' + runtime 'commons-discovery:commons-discovery:0.2' + compileOnly 'axis:axis:1.3' + runtime 'com.android.tools.build:builder:2.3.0' +} + +task sourcesJar(type: Jar, dependsOn: classes) { + classifier = 'sources' + from sourceSets.main.allSource +} + +task javadocJar(type: Jar, dependsOn: javadoc) { + classifier = 'javadoc' + from javadoc.destinationDir +} + +artifacts { + archives sourcesJar + archives javadocJar +} + +// To specify a license in the pom: +install { + repositories.mavenInstaller { + pom.project { + licenses { + license { + name 'The Apache Software License, Version 2.0' + url 'http://www.apache.org/licenses/LICENSE-2.0.txt' + distribution 'repo' + } + } + } + } +} diff --git a/test/find-files.test.ts b/test/find-files.test.ts index aa34873061c..d37558f26e9 100644 --- a/test/find-files.test.ts +++ b/test/find-files.test.ts @@ -6,7 +6,7 @@ const testFixture = path.join(__dirname, 'fixtures', 'find-files'); test('find all files in test fixture', async (t) => { // six levels deep to find all - const result = await find(testFixture, [], [], 6); + const { files: result, allFilesFound } = await find(testFixture, [], [], 6); const expected = [ path.join( testFixture, @@ -29,13 +29,46 @@ test('find all files in test fixture', async (t) => { path.join(testFixture, 'ruby', 'Gemfile.lock'), path.join(testFixture, 'yarn', 'yarn.lock'), ]; + const filteredOut = [ + path.join(testFixture, 'golang', 'golang-app', 'Gopkg.toml'), + path.join(testFixture, 'README.md'), + path.join(testFixture, 'yarn', 'package.json'), + path.join(testFixture, 'ruby', 'Gemfile'), + path.join(testFixture, 'gradle-kts', 'subproj', 'build.gradle.kts'), + path.join(testFixture, 'npm-with-lockfile', 'package.json'), + path.join(testFixture, 'gradle', 'subproject', 'build.gradle'), + path.join(testFixture, 'gradle-and-kotlin', 'build.gradle.kts'), + path.join( + testFixture, + 'gradle-multiple', + 'gradle', + 'subproject', + 'build.gradle', + ), + path.join( + testFixture, + 'gradle-multiple', + 'gradle-another', + 'subproject', + 'build.gradle', + ), + path.join(testFixture, 'maven', 'test.txt'), + path.join(testFixture, 'mvn', 'test.txt'), + path.join(testFixture, 'npm', 'test.txt'), + path.join(testFixture, 'ruby', 'test.txt'), + ]; t.same(result.length, expected.length, 'should be the same length'); t.same(result.sort(), expected.sort(), 'should return all files'); + t.same( + allFilesFound.filter((f) => !f.endsWith('broken-symlink')).sort(), + [...filteredOut, ...expected].sort(), + 'should return all unfiltered files', + ); }); test('find all files in test fixture ignoring node_modules', async (t) => { // six levels deep to ensure node_modules is tested - const result = await find(testFixture, ['node_modules'], [], 6); + const { files: result } = await find(testFixture, ['node_modules'], [], 6); const expected = [ path.join( testFixture, @@ -64,14 +97,19 @@ test('find all files in test fixture ignoring node_modules', async (t) => { test('find package.json file in test fixture ignoring node_modules', async (t) => { // six levels deep to ensure node_modules is tested const nodeModulesPath = path.join(testFixture, 'node_modules'); - const result = await find(nodeModulesPath, [], ['package.json'], 6); + const { files: result } = await find( + nodeModulesPath, + [], + ['package.json'], + 6, + ); const expected = []; t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find package.json file in test fixture (by default ignoring node_modules)', async (t) => { // six levels deep to ensure node_modules is tested - const result = await find(testFixture, [], ['package.json'], 6); + const { files: result } = await find(testFixture, [], ['package.json'], 6); const expected = [ path.join(testFixture, 'npm', 'package.json'), path.join(testFixture, 'npm-with-lockfile', 'package.json'), @@ -83,7 +121,7 @@ test('find package.json file in test fixture (by default ignoring node_modules)' test('find package-lock.json file in test fixture (ignore package.json in the same folder)', async (t) => { const npmLockfilePath = path.join(testFixture, 'npm-with-lockfile'); - const result = await find( + const { files: result } = await find( npmLockfilePath, [], ['package.json', 'package-lock.json'], @@ -96,7 +134,7 @@ test('find package-lock.json file in test fixture (ignore package.json in the sa test('find build.gradle file in test fixture (ignore build.gradle in the same folder)', async (t) => { const buildGradle = path.join(testFixture, 'gradle-and-kotlin'); - const result = await find( + const { files: result } = await find( buildGradle, [], ['build.gradle.kts', 'build.gradle'], @@ -109,7 +147,7 @@ test('find build.gradle file in test fixture (ignore build.gradle in the same fo test('find Gemfile.lock file in test fixture (ignore Gemfile in the same folder)', async (t) => { const npmLockfilePath = path.join(testFixture, 'ruby'); - const result = await find( + const { files: result } = await find( npmLockfilePath, [], ['Gemfile', 'Gemfile.lock'], @@ -122,7 +160,7 @@ test('find Gemfile.lock file in test fixture (ignore Gemfile in the same folder) test('find yarn.lock file in test fixture (ignore package.json in the same folder)', async (t) => { const yarnLockfilePath = path.join(testFixture, 'yarn'); - const result = await find( + const { files: result } = await find( yarnLockfilePath, [], ['package.json', 'yarn.lock'], @@ -134,7 +172,7 @@ test('find yarn.lock file in test fixture (ignore package.json in the same folde test('find package.json file in test fixture (by default ignoring node_modules)', async (t) => { // four levels deep to ensure node_modules is tested - const result = await find(testFixture, [], ['package.json'], 4); + const { files: result } = await find(testFixture, [], ['package.json'], 4); const expected = [ path.join(testFixture, 'npm', 'package.json'), path.join(testFixture, 'npm-with-lockfile', 'package.json'), @@ -144,13 +182,13 @@ test('find package.json file in test fixture (by default ignoring node_modules)' }); test('find Gemfile file in test fixture', async (t) => { - const result = await find(testFixture, [], ['Gemfile']); + const { files: result } = await find(testFixture, [], ['Gemfile']); const expected = [path.join(testFixture, 'ruby', 'Gemfile')]; t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find pom.xml files in test fixture', async (t) => { - const result = await find(testFixture, [], ['pom.xml']); + const { files: result } = await find(testFixture, [], ['pom.xml']); const expected = [ path.join(testFixture, 'maven', 'pom.xml'), path.join(testFixture, 'mvn', 'pom.xml'),