From cdffdc3a165e80bc175f2bba9ffee37a7d809073 Mon Sep 17 00:00:00 2001 From: Clinton Herget Date: Wed, 12 Aug 2020 16:06:52 -0400 Subject: [PATCH] fix: skip broken symlinks in find-files do not fail catastrophically on broken symlinks in project folder when using --all-projects --- src/cli/args.ts | 3 +++ src/lib/find-files.ts | 39 +++++++++++++++++++++-------------- src/lib/snyk-test/run-test.ts | 2 +- test/find-files.test.ts | 39 +++++++++++++---------------------- 4 files changed, 42 insertions(+), 41 deletions(-) diff --git a/src/cli/args.ts b/src/cli/args.ts index ed956f8720b..fb07fd8752c 100644 --- a/src/cli/args.ts +++ b/src/cli/args.ts @@ -32,6 +32,9 @@ alias.t = 'test'; const DEBUG_DEFAULT_NAMESPACES = [ 'snyk-test', 'snyk', + 'snyk:find-files', + 'snyk:run-test', + 'snyk:prune', 'snyk-gradle-plugin', 'snyk-sbt-plugin', 'snyk-mvn-plugin', diff --git a/src/lib/find-files.ts b/src/lib/find-files.ts index 01bf502834f..c89eb3b64a7 100644 --- a/src/lib/find-files.ts +++ b/src/lib/find-files.ts @@ -3,7 +3,7 @@ import * as pathLib from 'path'; import * as _ from '@snyk/lodash'; import { detectPackageManagerFromFile } from './detect'; import * as debugModule from 'debug'; -const debug = debugModule('snyk'); +const debug = debugModule('snyk:find-files'); // TODO: use util.promisify once we move to node 8 @@ -78,6 +78,7 @@ export async function find( found.push(fileFound); } } + return filterForDefaultManifests(found); } catch (err) { throw new Error(`Error finding files in path '${path}'.\n${err.message}`); @@ -107,6 +108,10 @@ async function findInDirectory( .filter((file) => !ignore.includes(file)) .map((file) => { const resolvedPath = pathLib.resolve(path, file); + if (!fs.existsSync(resolvedPath)) { + debug('File does not seem to exist, skipping: ', file); + return []; + } return find(resolvedPath, ignore, filter, levelsDeep); }); const found = await Promise.all(toFind); @@ -124,6 +129,7 @@ function filterForDefaultManifests(files: string[]): string[] { const foundFiles = _(files) .filter(Boolean) + .filter((p) => fs.existsSync(p)) .map((p) => ({ path: p, ...pathLib.parse(p), @@ -136,6 +142,7 @@ function filterForDefaultManifests(files: string[]): string[] { for (const directory of Object.keys(foundFiles)) { const filesInDirectory = foundFiles[directory]; const groupedFiles = _(filesInDirectory) + .filter((p) => !!p.packageManager) .groupBy('packageManager') .value(); @@ -217,52 +224,54 @@ function chooseBestManifest( const lockFile = files.filter((path) => ['package-lock.json', 'yarn.lock'].includes(path.base), )[0]; + debug( + `Encountered multiple node lockfiles files, defaulting to ${lockFile.path}`, + ); if (lockFile) { - debug( - 'Encountered multiple npm manifest files, defaulting to package-lock.json / yarn.lock', - ); return lockFile.path; } const packageJson = files.filter((path) => ['package.json'].includes(path.base), )[0]; debug( - 'Encountered multiple npm manifest files, defaulting to package.json', + `Encountered multiple npm manifest files, defaulting to ${packageJson.path}`, ); return packageJson.path; } case 'rubygems': { - debug( - 'Encountered multiple gem manifest files, defaulting to Gemfile.lock', - ); const defaultManifest = files.filter((path) => ['Gemfile.lock'].includes(path.base), )[0]; + debug( + `Encountered multiple gem manifest files, defaulting to ${defaultManifest.path}`, + ); return defaultManifest.path; } case 'cocoapods': { - debug( - 'Encountered multiple cocoapods manifest files, defaulting to Podfile', - ); const defaultManifest = files.filter((path) => ['Podfile'].includes(path.base), )[0]; + debug( + `Encountered multiple cocoapods manifest files, defaulting to ${defaultManifest.path}`, + ); return defaultManifest.path; } case 'pip': { - debug('Encountered multiple pip manifest files, defaulting to Pipfile'); const defaultManifest = files.filter((path) => ['Pipfile'].includes(path.base), )[0]; + debug( + `Encountered multiple pip manifest files, defaulting to ${defaultManifest.path}`, + ); return defaultManifest.path; } case 'gradle': { - debug( - 'Encountered multiple gradle manifest files, defaulting to build.gradle', - ); const defaultManifest = files.filter((path) => ['build.gradle'].includes(path.base), )[0]; + debug( + `Encountered multiple gradle manifest files, defaulting to ${defaultManifest.path}`, + ); return defaultManifest.path; } default: { diff --git a/src/lib/snyk-test/run-test.ts b/src/lib/snyk-test/run-test.ts index 740899893c0..7d29e3fc1fe 100644 --- a/src/lib/snyk-test/run-test.ts +++ b/src/lib/snyk-test/run-test.ts @@ -62,7 +62,7 @@ import * as alerts from '../alerts'; import { abridgeErrorMessage } from '../error-format'; import { getDockerToken } from '../api-token'; -const debug = debugModule('snyk'); +const debug = debugModule('snyk:run-test'); const ANALYTICS_PAYLOAD_MAX_LENGTH = 1024; diff --git a/test/find-files.test.ts b/test/find-files.test.ts index 052f30db080..aa34873061c 100644 --- a/test/find-files.test.ts +++ b/test/find-files.test.ts @@ -8,7 +8,6 @@ test('find all files in test fixture', async (t) => { // six levels deep to find all const result = await find(testFixture, [], [], 6); const expected = [ - path.join(testFixture, 'README.md'), path.join( testFixture, 'golang', @@ -17,7 +16,6 @@ test('find all files in test fixture', async (t) => { 'vendor.json', ), path.join(testFixture, 'golang', 'golang-app', 'Gopkg.lock'), - path.join(testFixture, 'golang', 'golang-app', 'Gopkg.toml'), path.join(testFixture, 'golang', 'golang-gomodules', 'go.mod'), path.join(testFixture, 'gradle', 'build.gradle'), path.join(testFixture, 'gradle-kts', 'build.gradle.kts'), @@ -25,24 +23,20 @@ test('find all files in test fixture', async (t) => { path.join(testFixture, 'gradle-multiple', 'gradle/build.gradle'), path.join(testFixture, 'gradle-multiple', 'gradle-another/build.gradle'), path.join(testFixture, 'maven', 'pom.xml'), - path.join(testFixture, 'maven', 'test.txt'), path.join(testFixture, 'npm-with-lockfile', 'package-lock.json'), path.join(testFixture, 'mvn', 'pom.xml'), - path.join(testFixture, 'mvn', 'test.txt'), path.join(testFixture, 'npm', 'package.json'), - path.join(testFixture, 'npm', 'test.txt'), path.join(testFixture, 'ruby', 'Gemfile.lock'), - path.join(testFixture, 'ruby', 'test.txt'), path.join(testFixture, 'yarn', 'yarn.lock'), - ].sort(); - t.same(result.sort(), expected, 'should return all files'); + ]; + t.same(result.length, expected.length, 'should be the same length'); + t.same(result.sort(), expected.sort(), 'should return all files'); }); test('find all files in test fixture ignoring node_modules', async (t) => { // six levels deep to ensure node_modules is tested const result = await find(testFixture, ['node_modules'], [], 6); const expected = [ - path.join(testFixture, 'README.md'), path.join( testFixture, 'golang', @@ -51,7 +45,6 @@ test('find all files in test fixture ignoring node_modules', async (t) => { 'vendor.json', ), path.join(testFixture, 'golang', 'golang-app', 'Gopkg.lock'), - path.join(testFixture, 'golang', 'golang-app', 'Gopkg.toml'), path.join(testFixture, 'golang', 'golang-gomodules', 'go.mod'), path.join(testFixture, 'gradle', 'build.gradle'), path.join(testFixture, 'gradle-kts', 'build.gradle.kts'), @@ -59,17 +52,13 @@ test('find all files in test fixture ignoring node_modules', async (t) => { path.join(testFixture, 'gradle-multiple', 'gradle/build.gradle'), path.join(testFixture, 'gradle-multiple', 'gradle-another/build.gradle'), path.join(testFixture, 'maven', 'pom.xml'), - path.join(testFixture, 'maven', 'test.txt'), path.join(testFixture, 'mvn', 'pom.xml'), - path.join(testFixture, 'mvn', 'test.txt'), path.join(testFixture, 'npm-with-lockfile', 'package-lock.json'), path.join(testFixture, 'npm', 'package.json'), - path.join(testFixture, 'npm', 'test.txt'), path.join(testFixture, 'ruby', 'Gemfile.lock'), - path.join(testFixture, 'ruby', 'test.txt'), path.join(testFixture, 'yarn', 'yarn.lock'), - ].sort(); - t.same(result.sort(), expected, 'should return expected files'); + ]; + t.same(result.sort(), expected.sort(), 'should return expected files'); }); test('find package.json file in test fixture ignoring node_modules', async (t) => { @@ -77,7 +66,7 @@ test('find package.json file in test fixture ignoring node_modules', async (t) = const nodeModulesPath = path.join(testFixture, 'node_modules'); const result = await find(nodeModulesPath, [], ['package.json'], 6); const expected = []; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find package.json file in test fixture (by default ignoring node_modules)', async (t) => { @@ -88,7 +77,7 @@ test('find package.json file in test fixture (by default ignoring node_modules)' path.join(testFixture, 'npm-with-lockfile', 'package.json'), path.join(testFixture, 'yarn', 'package.json'), ]; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find package-lock.json file in test fixture (ignore package.json in the same folder)', async (t) => { @@ -101,7 +90,7 @@ test('find package-lock.json file in test fixture (ignore package.json in the sa 1, ); const expected = [path.join(npmLockfilePath, 'package-lock.json')]; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find build.gradle file in test fixture (ignore build.gradle in the same folder)', async (t) => { @@ -114,7 +103,7 @@ test('find build.gradle file in test fixture (ignore build.gradle in the same fo 1, ); const expected = [path.join(buildGradle, 'build.gradle')]; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find Gemfile.lock file in test fixture (ignore Gemfile in the same folder)', async (t) => { @@ -127,7 +116,7 @@ test('find Gemfile.lock file in test fixture (ignore Gemfile in the same folder) 1, ); const expected = [path.join(npmLockfilePath, 'Gemfile.lock')]; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find yarn.lock file in test fixture (ignore package.json in the same folder)', async (t) => { @@ -140,7 +129,7 @@ test('find yarn.lock file in test fixture (ignore package.json in the same folde 1, ); const expected = [path.join(yarnLockfilePath, 'yarn.lock')]; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find package.json file in test fixture (by default ignoring node_modules)', async (t) => { @@ -151,13 +140,13 @@ test('find package.json file in test fixture (by default ignoring node_modules)' path.join(testFixture, 'npm-with-lockfile', 'package.json'), path.join(testFixture, 'yarn', 'package.json'), ]; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find Gemfile file in test fixture', async (t) => { const result = await find(testFixture, [], ['Gemfile']); const expected = [path.join(testFixture, 'ruby', 'Gemfile')]; - t.same(result, expected, 'should return expected file'); + t.same(result.sort(), expected.sort(), 'should return expected file'); }); test('find pom.xml files in test fixture', async (t) => { @@ -165,7 +154,7 @@ test('find pom.xml files in test fixture', async (t) => { const expected = [ path.join(testFixture, 'maven', 'pom.xml'), path.join(testFixture, 'mvn', 'pom.xml'), - ].sort(); + ]; t.same(result.sort(), expected, 'should return expected files'); });