Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
feat: introduce yarn workspaces scanning test & monitor
  • Loading branch information
lili2311 committed Jun 29, 2020
1 parent bc7dac7 commit 51c75d4
Show file tree
Hide file tree
Showing 17 changed files with 820 additions and 16 deletions.
1 change: 1 addition & 0 deletions src/cli/args.ts
Expand Up @@ -181,6 +181,7 @@ export function args(rawArgv: string[]): Args {
'scan-all-unmanaged',
'fail-on',
'all-projects',
'yarn-workspaces',
'detection-depth',
'reachable-vulns',
]) {
Expand Down
Expand Up @@ -22,9 +22,6 @@ export function formatMonitorOutput(
: res.path;
const strOutput =
chalk.bold.white('\nMonitoring ' + humanReadableName + '...\n\n') +
(packageManager === 'yarn'
? 'A yarn.lock file was detected - continuing as a Yarn project.\n'
: '') +
'Explore this snapshot at ' +
res.uri +
'\n\n' +
Expand Down
2 changes: 1 addition & 1 deletion src/cli/commands/monitor/index.ts
Expand Up @@ -120,7 +120,7 @@ async function monitor(...args0: MethodArgs): Promise<any> {
validateMonitorPath(path, options.docker);
let analysisType = 'all';
let packageManager;
if (options.allProjects) {
if (options.allProjects || options.yarnWorkspaces) {
analysisType = 'all';
} else if (options.docker) {
analysisType = 'docker';
Expand Down
6 changes: 6 additions & 0 deletions src/cli/index.ts
Expand Up @@ -244,6 +244,12 @@ async function main() {
if (args.options.file && args.options.allProjects) {
throw new UnsupportedOptionCombinationError(['file', 'all-projects']);
}
if (args.options.yarnWorkspaces && args.options.allProjects) {
throw new UnsupportedOptionCombinationError([
'yarn-workspaces',
'all-projects',
]);
}
if (args.options.packageManager && args.options.allProjects) {
throw new UnsupportedOptionCombinationError([
'package-manager',
Expand Down
2 changes: 1 addition & 1 deletion src/lib/options-validator.ts
Expand Up @@ -9,7 +9,7 @@ export async function validateOptions(
): Promise<void> {
if (options.reachableVulns) {
// Throwing error only in case when both packageManager and allProjects not defined
if (!packageManager && !options.allProjects) {
if (!packageManager && !options.allProjects && !options.yarnWorkspaces) {
throw new Error('Could not determine package manager');
}
const org = options.org || config.org;
Expand Down
25 changes: 21 additions & 4 deletions src/lib/plugins/get-deps-from-plugin.ts
Expand Up @@ -13,23 +13,36 @@ import {
import analytics = require('../analytics');
import { convertSingleResultToMultiCustom } from './convert-single-splugin-res-to-multi-custom';
import { convertMultiResultToMultiCustom } from './convert-multi-plugin-res-to-multi-custom';
import { processYarnWorkspaces } from './nodejs-plugin/yarn-workspaces-parser';

const debug = debugModule('snyk-test');

const multiProjectProcessors = {
yarnWorkspaces: {
handler: processYarnWorkspaces,
files: ['package.json'],
},
allProjects: {
handler: getMultiPluginResult,
files: AUTO_DETECTABLE_FILES,
},
};

// Force getDepsFromPlugin to return scannedProjects for processing
export async function getDepsFromPlugin(
root: string,
options: Options & (TestOptions | MonitorOptions),
): Promise<pluginApi.MultiProjectResult> {
let inspectRes: pluginApi.InspectResult;

if (options.allProjects) {
if (Object.keys(multiProjectProcessors).some((key) => options[key])) {
const scanType = options.yarnWorkspaces ? 'yarnWorkspaces' : 'allProjects';
const levelsDeep = options.detectionDepth;
const ignore = options.exclude ? options.exclude.split(',') : [];
const targetFiles = await find(
root,
ignore,
AUTO_DETECTABLE_FILES,
multiProjectProcessors[scanType].files,
levelsDeep,
);
debug(
Expand All @@ -39,7 +52,11 @@ export async function getDepsFromPlugin(
if (targetFiles.length === 0) {
throw NoSupportedManifestsFoundError([root]);
}
inspectRes = await getMultiPluginResult(root, options, targetFiles);
inspectRes = await multiProjectProcessors[scanType].handler(
root,
options,
targetFiles,
);
const analyticData = {
scannedProjects: inspectRes.scannedProjects.length,
targetFiles,
Expand All @@ -49,7 +66,7 @@ export async function getDepsFromPlugin(
levelsDeep,
ignore,
};
analytics.add('allProjects', analyticData);
analytics.add(scanType, analyticData);
return inspectRes;
}

Expand Down
136 changes: 136 additions & 0 deletions src/lib/plugins/nodejs-plugin/yarn-workspaces-parser.ts
@@ -0,0 +1,136 @@
import * as baseDebug from 'debug';
const debug = baseDebug('yarn-workspaces');
import * as fs from 'fs';
import * as lockFileParser from 'snyk-nodejs-lockfile-parser';
import * as path from 'path';
import { NoSupportedManifestsFoundError } from '../../errors';
import {
MultiProjectResultCustom,
ScannedProjectCustom,
} from '../get-multi-plugin-result';

export async function processYarnWorkspaces(
root: string,
settings: {
strictOutOfSync?: boolean;
scanDevDependencies?: boolean;
},
targetFiles: string[],
): Promise<MultiProjectResultCustom> {
const yarnTargetFiles = targetFiles.filter((f) => f.endsWith('package.json'));
debug(`processing Yarn workspaces (${targetFiles.length})`);
if (yarnTargetFiles.length === 0) {
throw NoSupportedManifestsFoundError([root]);
}
let yarnWorkspacesMap = {};
const yarnWorkspacesFilesMap = {};
let isYarnWorkspacePackage = false;
const result: MultiProjectResultCustom = {
plugin: {
name: 'snyk-nodejs-yarn-workspaces',
runtime: process.version,
},
scannedProjects: [],
};
for (const packageJsonFileName of yarnTargetFiles) {
const packageJson = getFileContents(root, packageJsonFileName);
yarnWorkspacesMap = {
...yarnWorkspacesMap,
...getWorkspacesMap(packageJson),
};

for (const workspaceRoot of Object.keys(yarnWorkspacesMap)) {
const workspaces = yarnWorkspacesMap[workspaceRoot].workspaces || [];
const match = workspaces
.map((pattern) =>
packageJsonFileName.includes(pattern.replace(/\*/, '')),
)
.filter(Boolean);

if (match) {
yarnWorkspacesFilesMap[packageJsonFileName] = {
root: workspaceRoot,
};
isYarnWorkspacePackage = true;
}
}

if (isYarnWorkspacePackage) {
const rootDir = path.dirname(
yarnWorkspacesFilesMap[packageJsonFileName].root,
);
const rootYarnLockfileName = path.join(rootDir, 'yarn.lock');

const yarnLock = await getFileContents(root, rootYarnLockfileName);
const res = await lockFileParser.buildDepTree(
packageJson.content,
yarnLock.content,
settings.scanDevDependencies,
lockFileParser.LockfileType.yarn,
settings.strictOutOfSync !== false,
);
const project: ScannedProjectCustom = {
packageManager: 'yarn',
targetFile: path.parse(packageJson.name).base,
depTree: res as any,
plugin: {
name: 'snyk-nodejs-lockfile-parser',
runtime: process.version,
},
};
result.scannedProjects.push(project);
}
}
return result;
}

function getFileContents(
root: string,
fileName: string,
): {
content: string;
name: string;
} {
const fullPath = path.resolve(root, fileName);
if (!fs.existsSync(fullPath)) {
throw new Error(
'Manifest ' + fileName + ' not found at location: ' + fileName,
);
}
const content = fs.readFileSync(fullPath, 'utf-8');
return {
content,
name: fileName,
};
}

interface YarnWorkspacesMap {
[packageJsonName: string]: {
workspaces: string[];
};
}

export function getWorkspacesMap(file: {
content: string;
name: string;
}): YarnWorkspacesMap {
const yarnWorkspacesMap = {};
if (!file) {
return yarnWorkspacesMap;
}

try {
const rootFileWorkspacesDefinitions = lockFileParser.getYarnWorkspaces(
file.content,
);

if (rootFileWorkspacesDefinitions && rootFileWorkspacesDefinitions.length) {
yarnWorkspacesMap[file.name] = {
workspaces: rootFileWorkspacesDefinitions,
};
}
} catch (e) {
debug('Failed to process a workspace', e.message);
}
return yarnWorkspacesMap;
}
1 change: 1 addition & 0 deletions src/lib/reachable-vulns.ts
Expand Up @@ -37,6 +37,7 @@ export async function validatePayload(
if (
packageManager &&
!options.allProjects &&
!options.yarnWorkspaces &&
!REACHABLE_VULNS_SUPPORTED_PACKAGE_MANAGERS.includes(packageManager)
) {
throw new FeatureNotSupportedByPackageManagerError(
Expand Down
9 changes: 4 additions & 5 deletions src/lib/snyk-test/index.js
Expand Up @@ -32,11 +32,9 @@ async function test(root, options, callback) {
function executeTest(root, options) {
try {
if (!options.allProjects) {
if (options.iac) {
options.packageManager = detect.isIacProject(root, options);
} else {
options.packageManager = detect.detectPackageManager(root, options);
}
options.packageManager = options.iac
? detect.isIacProject(root, options)
: detect.detectPackageManager(root, options);
}
return run(root, options).then((results) => {
for (const res of results) {
Expand Down Expand Up @@ -70,6 +68,7 @@ function run(root, options) {
!(
options.docker ||
options.allProjects ||
options.yarnWorkspaces ||
pm.SUPPORTED_PACKAGE_MANAGER_NAME[packageManager]
)
) {
Expand Down
4 changes: 2 additions & 2 deletions src/lib/snyk-test/run-test.ts
Expand Up @@ -190,10 +190,10 @@ async function parseRes(
pkgManager,
options.severityThreshold,
);

// For Node.js: inject additional information (for remediation etc.) into the response.
if (payload.modules) {
res.dependencyCount = payload.modules.numDependencies;
res.dependencyCount =
payload.modules.numDependencies || depGraph.getPkgs().length - 1;
if (res.vulnerabilities) {
res.vulnerabilities.forEach((vuln) => {
if (payload.modules && payload.modules.pluck) {
Expand Down
3 changes: 3 additions & 0 deletions src/lib/types.ts
Expand Up @@ -19,6 +19,7 @@ export interface TestOptions {
showVulnPaths: ShowVulnPaths;
failOn?: FailOn;
reachableVulns?: boolean;
yarnWorkspaces?: boolean;
}

export interface WizardOptions {
Expand Down Expand Up @@ -63,6 +64,7 @@ export interface Options {
allProjects?: boolean;
detectionDepth?: number;
exclude?: string;
strictOutOfSync?: boolean;
// Used with the Docker plugin only. Allows requesting some experimental/unofficial features.
experimental?: boolean;
}
Expand All @@ -85,6 +87,7 @@ export interface MonitorOptions {
// Used with the Docker plugin only. Allows requesting some experimental/unofficial features.
experimental?: boolean;
reachableVulns?: boolean;
yarnWorkspaces?: boolean;
}

export interface MonitorMeta {
Expand Down
1 change: 1 addition & 0 deletions test/acceptance/cli-args.test.ts
Expand Up @@ -120,6 +120,7 @@ const argsNotAllowedWithAllProjects = [
'project-name',
'docker',
'all-sub-projects',
'yarn-workspaces',
];

argsNotAllowedWithAllProjects.forEach((arg) => {
Expand Down

0 comments on commit 51c75d4

Please sign in to comment.