diff --git a/CHANGELOG.md b/CHANGELOG.md index 3013ef545b..e8cae7e411 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,10 @@ Please add one entry in this file for each change in Yarn's behavior. Use the sa [#6447](https://github.com/yarnpkg/yarn/pull/6447) - [**John-David Dalton**](https://twitter.com/jdalton) +- Adds `yarn audit` (and the `--audit` flag for all installs) + + [#6409](https://github.com/yarnpkg/yarn/pull/6409) - [**Jeff Valore**](https://github.com/rally25rs) + - Adds a special logic to PnP for ESLint compatibility (temporary, until [eslint/eslint#10125](https://github.com/eslint/eslint/issues/10125) is fixed) [#6449](https://github.com/yarnpkg/yarn/pull/6449) - [**Maël Nison**](https://twitter.com/arcanis) diff --git a/__tests__/commands/audit.js b/__tests__/commands/audit.js new file mode 100644 index 0000000000..aaf0fa5046 --- /dev/null +++ b/__tests__/commands/audit.js @@ -0,0 +1,125 @@ +/* @flow */ + +import {NoopReporter} from '../../src/reporters/index.js'; +import {run as buildRun} from './_helpers.js'; +import {run as audit} from '../../src/cli/commands/audit.js'; +import {promisify} from '../../src/util/promise.js'; + +const path = require('path'); +const zlib = require('zlib'); +const gunzip = promisify(zlib.gunzip); + +const fixturesLoc = path.join(__dirname, '..', 'fixtures', 'audit'); + +const setupMockRequestManager = function(config) { + const apiResponse = JSON.stringify(getAuditResponse(config), null, 2); + // $FlowFixMe + config.requestManager.request = jest.fn(); + config.requestManager.request.mockReturnValue( + new Promise(resolve => { + resolve(apiResponse); + }), + ); +}; + +const setupMockReporter = function(reporter) { + // $FlowFixMe + reporter.auditAdvisory = jest.fn(); + // $FlowFixMe + reporter.auditAction = jest.fn(); + // $FlowFixMe + reporter.auditSummary = jest.fn(); +}; + +const getAuditResponse = function(config): Object { + // $FlowFixMe + return require(path.join(config.cwd, 'audit-api-response.json')); +}; + +const runAudit = buildRun.bind( + null, + NoopReporter, + fixturesLoc, + async (args, flags, config, reporter, lockfile, getStdout): Promise => { + setupMockRequestManager(config); + setupMockReporter(reporter); + await audit(config, reporter, flags, args); + return getStdout(); + }, +); + +test.concurrent('sends correct dependency map to audit api for single dependency.', () => { + const expectedApiPost = { + name: 'yarn-test', + install: [], + remove: [], + metadata: {}, + requires: { + minimatch: '^3.0.0', + }, + dependencies: { + minimatch: { + version: '3.0.0', + integrity: 'sha1-UjYVelHk8ATBd/s8Un/33Xjw74M=', + requires: { + 'brace-expansion': '^1.0.0', + }, + dependencies: {}, + }, + 'brace-expansion': { + version: '1.1.11', + integrity: 'sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA==', + requires: { + 'balanced-match': '^1.0.0', + 'concat-map': '0.0.1', + }, + dependencies: {}, + }, + 'balanced-match': { + version: '1.0.0', + integrity: 'sha1-ibTRmasr7kneFk6gK4nORi1xt2c=', + requires: {}, + dependencies: {}, + }, + 'concat-map': { + version: '0.0.1', + integrity: 'sha1-2Klr13/Wjfd5OnMDajug1UBdR3s=', + requires: {}, + dependencies: {}, + }, + }, + version: '0.0.0', + }; + + return runAudit([], {}, 'single-vulnerable-dep-installed', async config => { + const calledWithPipe = config.requestManager.request.mock.calls[0][0].body; + const calledWith = JSON.parse(await gunzip(calledWithPipe)); + expect(calledWith).toEqual(expectedApiPost); + }); +}); + +test('calls reporter auditAdvisory with correct data', () => { + return runAudit([], {}, 'single-vulnerable-dep-installed', (config, reporter) => { + const apiResponse = getAuditResponse(config); + expect(reporter.auditAdvisory).toBeCalledWith(apiResponse.actions[0].resolves[0], apiResponse.advisories['118']); + }); +}); + +// *** Test temporarily removed due to inability to correctly puggest actions to the user. +// test('calls reporter auditAction with correct data', () => { +// return runAudit([], {}, 'single-vulnerable-dep-installed', (config, reporter) => { +// const apiResponse = getAuditResponse(config); +// expect(reporter.auditAction).toBeCalledWith({ +// cmd: 'yarn upgrade minimatch@3.0.4', +// isBreaking: false, +// action: apiResponse.actions[0], +// }); +// }); +// }); + +test('calls reporter auditSummary with correct data', () => { + return runAudit([], {}, 'single-vulnerable-dep-installed', (config, reporter) => { + const apiResponse = getAuditResponse(config); + expect(reporter.auditSummary).toBeCalledWith(apiResponse.metadata); + }); +}); diff --git a/__tests__/fixtures/audit/single-vulnerable-dep-installed/audit-api-response.json b/__tests__/fixtures/audit/single-vulnerable-dep-installed/audit-api-response.json new file mode 100644 index 0000000000..e0d4605b3e --- /dev/null +++ b/__tests__/fixtures/audit/single-vulnerable-dep-installed/audit-api-response.json @@ -0,0 +1,77 @@ +{ + "actions": [ + { + "action": "install", + "module": "minimatch", + "target": "3.0.4", + "isMajor": false, + "resolves": [ + { + "id": 118, + "path": "minimatch", + "dev": false, + "optional": false, + "bundled": false + } + ] + } + ], + "advisories": { + "118": { + "findings": [ + { + "version": "3.0.0", + "paths": [ + "minimatch" + ], + "dev": false, + "optional": false, + "bundled": false + } + ], + "id": 118, + "created": "2016-05-25T16:37:20.000Z", + "updated": "2018-03-01T21:58:01.072Z", + "deleted": null, + "title": "Regular Expression Denial of Service", + "found_by": { + "name": "Nick Starke" + }, + "reported_by": { + "name": "Nick Starke" + }, + "module_name": "minimatch", + "cves": [ + "CVE-2016-10540" + ], + "vulnerable_versions": "<=3.0.1", + "patched_versions": ">=3.0.2", + "overview": "Affected versions of `minimatch` are vulnerable to regular expression denial of service attacks when user input is passed into the `pattern` argument of `minimatch(path, pattern)`.\n\n\n## Proof of Concept\n```\nvar minimatch = require(“minimatch”);\n\n// utility function for generating long strings\nvar genstr = function (len, chr) {\n var result = “”;\n for (i=0; i<=len; i++) {\n result = result + chr;\n }\n return result;\n}\n\nvar exploit = “[!” + genstr(1000000, “\\\\”) + “A”;\n\n// minimatch exploit.\nconsole.log(“starting minimatch”);\nminimatch(“foo”, exploit);\nconsole.log(“finishing minimatch”);\n```", + "recommendation": "Update to version 3.0.2 or later.", + "references": "", + "access": "public", + "severity": "high", + "cwe": "CWE-400", + "metadata": { + "module_type": "Multi.Library", + "exploitability": 4, + "affected_components": "Internal::Code::Function::minimatch({type:'args', key:0, vector:{type:'string'}})" + }, + "url": "https://nodesecurity.io/advisories/118" + } + }, + "muted": [], + "metadata": { + "vulnerabilities": { + "info": 0, + "low": 0, + "moderate": 0, + "high": 1, + "critical": 0 + }, + "dependencies": 5, + "devDependencies": 0, + "optionalDependencies": 0, + "totalDependencies": 5 + } +} diff --git a/__tests__/fixtures/audit/single-vulnerable-dep-installed/package.json b/__tests__/fixtures/audit/single-vulnerable-dep-installed/package.json new file mode 100644 index 0000000000..e9a954d98c --- /dev/null +++ b/__tests__/fixtures/audit/single-vulnerable-dep-installed/package.json @@ -0,0 +1,7 @@ +{ + "name": "yarn-test", + "version": "0.0.0", + "dependencies": { + "minimatch": "^3.0.0" + } +} diff --git a/__tests__/fixtures/audit/single-vulnerable-dep-installed/yarn.lock b/__tests__/fixtures/audit/single-vulnerable-dep-installed/yarn.lock new file mode 100644 index 0000000000..ec5ea84a83 --- /dev/null +++ b/__tests__/fixtures/audit/single-vulnerable-dep-installed/yarn.lock @@ -0,0 +1,28 @@ +# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY. +# yarn lockfile v1 + + +balanced-match@^1.0.0: + version "1.0.0" + resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767" + integrity sha1-ibTRmasr7kneFk6gK4nORi1xt2c= + +brace-expansion@^1.0.0: + version "1.1.11" + resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd" + integrity sha512-iCuPHDFgrHX7H2vEI/5xpz07zSHB00TpugqhmYtVmMO6518mCuRMoOYFldEBl0g187ufozdaHgWKcYFb61qGiA== + dependencies: + balanced-match "^1.0.0" + concat-map "0.0.1" + +concat-map@0.0.1: + version "0.0.1" + resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" + integrity sha1-2Klr13/Wjfd5OnMDajug1UBdR3s= + +minimatch@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.0.tgz#5236157a51e4f004c177fb3c527ff7dd78f0ef83" + integrity sha1-UjYVelHk8ATBd/s8Un/33Xjw74M= + dependencies: + brace-expansion "^1.0.0" diff --git a/__tests__/reporters/__snapshots__/console-reporter.js.snap b/__tests__/reporters/__snapshots__/console-reporter.js.snap index a5352c97fb..05efa5069c 100644 --- a/__tests__/reporters/__snapshots__/console-reporter.js.snap +++ b/__tests__/reporters/__snapshots__/console-reporter.js.snap @@ -7,6 +7,14 @@ Object { } `; +exports[`ConsoleReporter.auditSummary 1`] = ` +Object { + "stderr": "", + "stdout": "1 vulnerabilities found - Packages audited: 5 +Severity: 1 High", +} +`; + exports[`ConsoleReporter.command 1`] = ` Object { "stderr": "", diff --git a/__tests__/reporters/__snapshots__/json-reporter.js.snap b/__tests__/reporters/__snapshots__/json-reporter.js.snap index 22ae133dd3..9606dd900a 100644 --- a/__tests__/reporters/__snapshots__/json-reporter.js.snap +++ b/__tests__/reporters/__snapshots__/json-reporter.js.snap @@ -17,6 +17,27 @@ Object { } `; +exports[`JSONReporter.auditAction 1`] = ` +Object { + "stderr": "", + "stdout": "{\\"type\\":\\"auditAction\\",\\"data\\":{\\"cmd\\":\\"yarn upgrade gulp@4.0.0\\",\\"isBreaking\\":true,\\"action\\":{\\"action\\":\\"install\\",\\"module\\":\\"gulp\\",\\"target\\":\\"4.0.0\\",\\"isMajor\\":true,\\"resolves\\":[]}}}", +} +`; + +exports[`JSONReporter.auditAdvisory 1`] = ` +Object { + "stderr": "", + "stdout": "{\\"type\\":\\"auditAdvisory\\",\\"data\\":{\\"resolution\\":{\\"id\\":118,\\"path\\":\\"gulp>vinyl-fs>glob-stream>minimatch\\",\\"dev\\":false,\\"optional\\":false,\\"bundled\\":false},\\"advisory\\":{\\"findings\\":[{\\"bundled\\":false,\\"optional\\":false,\\"dev\\":false,\\"paths\\":[],\\"version\\":\\"\\"}],\\"id\\":118,\\"created\\":\\"2016-05-25T16:37:20.000Z\\",\\"updated\\":\\"2018-03-01T21:58:01.072Z\\",\\"deleted\\":null,\\"title\\":\\"Regular Expression Denial of Service\\",\\"found_by\\":{\\"name\\":\\"Nick Starke\\"},\\"reported_by\\":{\\"name\\":\\"Nick Starke\\"},\\"module_name\\":\\"minimatch\\",\\"cves\\":[\\"CVE-2016-10540\\"],\\"vulnerable_versions\\":\\"<=3.0.1\\",\\"patched_versions\\":\\">=3.0.2\\",\\"overview\\":\\"\\",\\"recommendation\\":\\"Update to version 3.0.2 or later.\\",\\"references\\":\\"\\",\\"access\\":\\"public\\",\\"severity\\":\\"high\\",\\"cwe\\":\\"CWE-400\\",\\"metadata\\":{\\"module_type\\":\\"Multi.Library\\",\\"exploitability\\":4,\\"affected_components\\":\\"\\"},\\"url\\":\\"https://nodesecurity.io/advisories/118\\"}}}", +} +`; + +exports[`JSONReporter.auditSummary 1`] = ` +Object { + "stderr": "", + "stdout": "{\\"type\\":\\"auditSummary\\",\\"data\\":{\\"vulnerabilities\\":{\\"info\\":0,\\"low\\":1,\\"moderate\\":0,\\"high\\":4,\\"critical\\":0},\\"dependencies\\":29105,\\"devDependencies\\":0,\\"optionalDependencies\\":0,\\"totalDependencies\\":29105}}", +} +`; + exports[`JSONReporter.command 1`] = ` Object { "stderr": "", diff --git a/__tests__/reporters/console-reporter.js b/__tests__/reporters/console-reporter.js index 6a8a920039..2e52373825 100644 --- a/__tests__/reporters/console-reporter.js +++ b/__tests__/reporters/console-reporter.js @@ -304,3 +304,25 @@ test('ConsoleReporter.tree is silent when isSilent is true', async () => { }), ).toMatchSnapshot(); }); + +test('ConsoleReporter.auditSummary', async () => { + const auditMetadata = { + vulnerabilities: { + info: 0, + low: 0, + moderate: 0, + high: 1, + critical: 0, + }, + dependencies: 5, + devDependencies: 0, + optionalDependencies: 0, + totalDependencies: 5, + }; + + expect( + await getConsoleBuff(r => { + r.auditSummary(auditMetadata); + }), + ).toMatchSnapshot(); +}); diff --git a/__tests__/reporters/json-reporter.js b/__tests__/reporters/json-reporter.js index d1ee1183e6..f21b210f20 100644 --- a/__tests__/reporters/json-reporter.js +++ b/__tests__/reporters/json-reporter.js @@ -111,3 +111,91 @@ test('JSONReporter.progress', async () => { }), ).toMatchSnapshot(); }); + +test('JSONReporter.auditAction', async () => { + expect( + await getJSONBuff(r => { + r.auditAction({ + cmd: 'yarn upgrade gulp@4.0.0', + isBreaking: true, + action: { + action: 'install', + module: 'gulp', + target: '4.0.0', + isMajor: true, + resolves: [], + }, + }); + }), + ).toMatchSnapshot(); +}); + +test('JSONReporter.auditAdvisory', async () => { + expect( + await getJSONBuff(r => { + r.auditAdvisory( + { + id: 118, + path: 'gulp>vinyl-fs>glob-stream>minimatch', + dev: false, + optional: false, + bundled: false, + }, + { + findings: [ + { + bundled: false, + optional: false, + dev: false, + paths: [], + version: '', + }, + ], + id: 118, + created: '2016-05-25T16:37:20.000Z', + updated: '2018-03-01T21:58:01.072Z', + deleted: null, + title: 'Regular Expression Denial of Service', + found_by: {name: 'Nick Starke'}, + reported_by: {name: 'Nick Starke'}, + module_name: 'minimatch', + cves: ['CVE-2016-10540'], + vulnerable_versions: '<=3.0.1', + patched_versions: '>=3.0.2', + overview: '', + recommendation: 'Update to version 3.0.2 or later.', + references: '', + access: 'public', + severity: 'high', + cwe: 'CWE-400', + metadata: { + module_type: 'Multi.Library', + exploitability: 4, + affected_components: '', + }, + url: 'https://nodesecurity.io/advisories/118', + }, + ); + }), + ).toMatchSnapshot(); +}); + +test('JSONReporter.auditSummary', async () => { + expect( + await getJSONBuff(r => { + r.auditSummary({ + vulnerabilities: { + info: 0, + low: 1, + moderate: 0, + high: 4, + critical: 0, + }, + dependencies: 29105, + devDependencies: 0, + optionalDependencies: 0, + totalDependencies: 29105, + }); + }), + ).toMatchSnapshot(); +}); diff --git a/package.json b/package.json index edd70faa90..f7da64df89 100644 --- a/package.json +++ b/package.json @@ -11,6 +11,7 @@ "bytes": "^3.0.0", "camelcase": "^4.0.0", "chalk": "^2.1.0", + "cli-table3": "^0.5.1", "commander": "^2.9.0", "death": "^1.0.0", "debug": "^3.0.0", diff --git a/src/cli/commands/add.js b/src/cli/commands/add.js index 347af7f4c2..2ca65a41ce 100644 --- a/src/cli/commands/add.js +++ b/src/cli/commands/add.js @@ -300,6 +300,7 @@ export function setFlags(commander: Object) { commander.option('-O, --optional', 'save package to your `optionalDependencies`'); commander.option('-E, --exact', 'install exact version'); commander.option('-T, --tilde', 'install most recent release with the same minor version'); + commander.option('-A', '--audit', 'Run vulnerability audit on installed packages'); } export async function run(config: Config, reporter: Reporter, flags: Object, args: Array): Promise { diff --git a/src/cli/commands/audit.js b/src/cli/commands/audit.js new file mode 100644 index 0000000000..b9faa70381 --- /dev/null +++ b/src/cli/commands/audit.js @@ -0,0 +1,291 @@ +/* @flow */ + +import type Config from '../../config.js'; +import type PackageResolver from '../../package-resolver.js'; +import type PackageLinker from '../../package-linker.js'; +import type {Reporter} from '../../reporters/index.js'; +import type {HoistedTrees} from '../../hoisted-tree-builder.js'; + +import {promisify} from '../../util/promise.js'; +import {buildTree as hoistedTreeBuilder} from '../../hoisted-tree-builder'; +import {Install} from './install.js'; +import Lockfile from '../../lockfile'; +import {YARN_REGISTRY} from '../../constants'; + +const zlib = require('zlib'); +const gzip = promisify(zlib.gzip); + +export type AuditNode = { + version: ?string, + integrity: ?string, + requires: Object, + dependencies: {[string]: AuditNode}, +}; + +export type AuditTree = AuditNode & { + install: Array, + remove: Array, + metadata: Object, +}; + +export type AuditVulnerabilityCounts = { + info: number, + low: number, + moderate: number, + high: number, + critical: number, +}; + +export type AuditResolution = { + id: number, + path: string, + dev: boolean, + optional: boolean, + bundled: boolean, +}; + +export type AuditAction = { + action: string, + module: string, + target: string, + isMajor: boolean, + resolves: Array, +}; + +export type AuditAdvisory = { + findings: [ + { + version: string, + paths: Array, + dev: boolean, + optional: boolean, + bundled: boolean, + }, + ], + id: number, + created: string, + updated: string, + deleted: ?boolean, + title: string, + found_by: { + name: string, + }, + reported_by: { + name: string, + }, + module_name: string, + cves: Array, + vulnerable_versions: string, + patched_versions: string, + overview: string, + recommendation: string, + references: string, + access: string, + severity: string, + cwe: string, + metadata: { + module_type: string, + exploitability: number, + affected_components: string, + }, + url: string, +}; + +export type AuditMetadata = { + vulnerabilities: AuditVulnerabilityCounts, + dependencies: number, + devDependencies: number, + optionalDependencies: number, + totalDependencies: number, +}; + +export type AuditReport = { + actions: Array, + advisories: {[string]: AuditAdvisory}, + muted: Array, + metadata: AuditMetadata, +}; + +export type AuditActionRecommendation = { + cmd: string, + isBreaking: boolean, + action: AuditAction, +}; + +export function setFlags(commander: Object) { + commander.description('Checks for known security issues with the installed packages.'); + commander.option('--summary', 'Only print the summary.'); +} + +export function hasWrapper(commander: Object, args: Array): boolean { + return true; +} + +export async function run(config: Config, reporter: Reporter, flags: Object, args: Array): Promise { + const audit = new Audit(config, reporter); + const lockfile = await Lockfile.fromDirectory(config.lockfileFolder, reporter); + const install = new Install({}, config, reporter, lockfile); + const {manifest, requests, patterns, workspaceLayout} = await install.fetchRequestFromCwd(); + await install.resolver.init(requests, { + workspaceLayout, + }); + + const vulnerabilities = await audit.performAudit(manifest, install.resolver, install.linker, patterns); + const totalVulnerabilities = + vulnerabilities.info + + vulnerabilities.low + + vulnerabilities.moderate + + vulnerabilities.high + + vulnerabilities.critical; + + if (flags.summary) { + audit.summary(); + } else { + audit.report(); + } + + return totalVulnerabilities; +} + +export default class Audit { + constructor(config: Config, reporter: Reporter) { + this.config = config; + this.reporter = reporter; + } + + config: Config; + reporter: Reporter; + auditData: AuditReport; + + _mapHoistedNodes(auditNode: AuditNode, hoistedNodes: HoistedTrees) { + for (const node of hoistedNodes) { + const pkg = node.manifest.pkg; + auditNode.dependencies[node.name] = { + version: node.version, + integrity: pkg._remote ? pkg._remote.integrity || '' : '', + requires: Object.assign({}, pkg.dependencies || {}, pkg.optionalDependencies || {}), + dependencies: {}, + }; + + if (node.children) { + this._mapHoistedNodes(auditNode.dependencies[node.name], node.children); + } + } + } + + _mapHoistedTreesToAuditTree(manifest: Object, hoistedTrees: HoistedTrees): AuditTree { + const auditTree: AuditTree = { + name: manifest.name, + version: manifest.version || undefined, + install: [], + remove: [], + metadata: { + //TODO: What do we send here? npm sends npm version, node version, etc. + }, + requires: Object.assign( + {}, + manifest.dependencies || {}, + manifest.devDependencies || {}, + manifest.optionalDependencies || {}, + ), + integrity: undefined, + dependencies: {}, + }; + + this._mapHoistedNodes(auditTree, hoistedTrees); + return auditTree; + } + + async _fetchAudit(auditTree: AuditTree): Object { + let responseJson; + const registry = YARN_REGISTRY; + this.reporter.verbose(`Audit Request: ${JSON.stringify(auditTree, null, 2)}`); + const requestBody = await gzip(JSON.stringify(auditTree)); + const response = await this.config.requestManager.request({ + url: `${registry}/-/npm/v1/security/audits`, + method: 'POST', + body: requestBody, + headers: { + 'Content-Encoding': 'gzip', + 'Content-Type': 'application/json', + Accept: 'application/json', + }, + }); + + try { + responseJson = JSON.parse(response); + } catch (ex) { + throw new Error(`Unexpected audit response (Invalid JSON): ${response}`); + } + if (!responseJson.metadata) { + throw new Error(`Unexpected audit response (Missing Metadata): ${JSON.stringify(responseJson, null, 2)}`); + } + this.reporter.verbose(`Audit Response: ${JSON.stringify(responseJson, null, 2)}`); + return responseJson; + } + + async performAudit( + manifest: Object, + resolver: PackageResolver, + linker: PackageLinker, + patterns: Array, + ): Promise { + const hoistedTrees = await hoistedTreeBuilder(resolver, linker, patterns); + const auditTree = this._mapHoistedTreesToAuditTree(manifest, hoistedTrees); + this.auditData = await this._fetchAudit(auditTree); + return this.auditData.metadata.vulnerabilities; + } + + summary() { + if (!this.auditData) { + return; + } + this.reporter.auditSummary(this.auditData.metadata); + } + + report() { + if (!this.auditData) { + return; + } + + const reportAdvisory = (resolution: AuditResolution) => { + const advisory = this.auditData.advisories[resolution.id.toString()]; + this.reporter.auditAdvisory(resolution, advisory); + }; + + if (Object.keys(this.auditData.advisories).length !== 0) { + // let printedManualReviewHeader = false; + + this.auditData.actions.forEach(action => { + action.resolves.forEach(reportAdvisory); + + /* The following block has been temporarily removed + * because the actions returned by npm are not valid for yarn. + * Removing this action reporting until we can come up with a way + * to correctly resolve issues. + */ + // if (action.action === 'update' || action.action === 'install') { + // // these advisories can be resolved automatically by running a yarn command + // const recommendation: AuditActionRecommendation = { + // cmd: `yarn upgrade ${action.module}@${action.target}`, + // isBreaking: action.isMajor, + // action, + // }; + // this.reporter.auditAction(recommendation); + // action.resolves.forEach(reportAdvisory); + // } + + // if (action.action === 'review') { + // // these advisories cannot be resolved automatically and require manual review + // if (!printedManualReviewHeader) { + // this.reporter.auditManualReview(); + // } + // printedManualReviewHeader = true; + // action.resolves.forEach(reportAdvisory); + // } + }); + } + + this.summary(); + } +} diff --git a/src/cli/commands/index.js b/src/cli/commands/index.js index e162956ef4..182f9958aa 100644 --- a/src/cli/commands/index.js +++ b/src/cli/commands/index.js @@ -8,6 +8,7 @@ const getDocsInfo = name => 'Visit ' + chalk.bold(getDocsLink(name)) + ' for doc import * as access from './access.js'; import * as add from './add.js'; +import * as audit from './audit.js'; import * as autoclean from './autoclean.js'; import * as bin from './bin.js'; import * as cache from './cache.js'; @@ -51,6 +52,7 @@ import buildUseless from './_useless.js'; const commands = { access, add, + audit, autoclean, bin, cache, diff --git a/src/cli/commands/install.js b/src/cli/commands/install.js index 51baa45c89..ce3b392894 100644 --- a/src/cli/commands/install.js +++ b/src/cli/commands/install.js @@ -30,6 +30,7 @@ import {generatePnpMap} from '../../util/generate-pnp-map.js'; import WorkspaceLayout from '../../workspace-layout.js'; import ResolutionMap from '../../resolution-map.js'; import guessName from '../../util/guess-name'; +import Audit from './audit'; const deepEqual = require('deep-equal'); const emoji = require('node-emoji'); @@ -65,6 +66,7 @@ type Flags = { frozenLockfile: boolean, skipIntegrityCheck: boolean, checkFiles: boolean, + audit: boolean, // add peer: boolean, @@ -143,6 +145,7 @@ function normalizeFlags(config: Config, rawFlags: Object): Flags { frozenLockfile: !!rawFlags.frozenLockfile, linkDuplicates: !!rawFlags.linkDuplicates, checkFiles: !!rawFlags.checkFiles, + audit: !!rawFlags.audit, // add peer: !!rawFlags.peer, @@ -429,7 +432,16 @@ export class Install { return patterns; } + async prepareManifests(): Promise { + const manifests = await this.config.getRootManifests(); + return manifests; + } + async bailout(patterns: Array, workspaceLayout: ?WorkspaceLayout): Promise { + // We don't want to skip the audit - it could yield important errors + if (this.flags.audit) { + return false; + } // PNP is so fast that the integrity check isn't pertinent if (this.config.plugnplayEnabled) { return false; @@ -565,6 +577,9 @@ export class Install { }); } + const audit = new Audit(this.config, this.reporter); + let auditFoundProblems = false; + steps.push((curr: number, total: number) => callThroughHook('resolveStep', async () => { this.reporter.step(curr, total, this.reporter.lang('resolvingPackages'), emoji.get('mag')); @@ -575,10 +590,38 @@ export class Install { }); topLevelPatterns = this.preparePatterns(rawPatterns); flattenedTopLevelPatterns = await this.flatten(topLevelPatterns); - return {bailout: await this.bailout(topLevelPatterns, workspaceLayout)}; + return {bailout: !this.flags.audit && (await this.bailout(topLevelPatterns, workspaceLayout))}; }), ); + if (this.flags.audit) { + steps.push((curr: number, total: number) => + callThroughHook('auditStep', async () => { + this.reporter.step(curr, total, this.reporter.lang('auditRunning'), emoji.get('mag')); + if (this.flags.offline) { + this.reporter.warn(this.reporter.lang('auditOffline')); + return {bailout: false}; + } + const preparedManifests = await this.prepareManifests(); + // $FlowFixMe - Flow considers `m` in the map operation to be "mixed", so does not recognize `m.object` + const mergedManifest = Object.assign({}, ...Object.values(preparedManifests).map(m => m.object)); + const auditVulnerabilityCounts = await audit.performAudit( + mergedManifest, + this.resolver, + this.linker, + topLevelPatterns, + ); + auditFoundProblems = + auditVulnerabilityCounts.info || + auditVulnerabilityCounts.low || + auditVulnerabilityCounts.moderate || + auditVulnerabilityCounts.high || + auditVulnerabilityCounts.critical; + return {bailout: await this.bailout(topLevelPatterns, workspaceLayout)}; + }), + ); + } + steps.push((curr: number, total: number) => callThroughHook('fetchStep', async () => { this.markIgnored(ignorePatterns); @@ -673,12 +716,24 @@ export class Install { for (const step of steps) { const stepResult = await step(++currentStep, steps.length); if (stepResult && stepResult.bailout) { + if (this.flags.audit) { + audit.summary(); + } + if (auditFoundProblems) { + this.reporter.warn(this.reporter.lang('auditRunAuditForDetails')); + } this.maybeOutputUpdate(); return flattenedTopLevelPatterns; } } // fin! + if (this.flags.audit) { + audit.summary(); + } + if (auditFoundProblems) { + this.reporter.warn(this.reporter.lang('auditRunAuditForDetails')); + } await this.saveLockfileAndIntegrity(topLevelPatterns, workspaceLayout); await this.persistChanges(); this.maybeOutputUpdate(); @@ -1067,6 +1122,7 @@ export function hasWrapper(commander: Object, args: Array): boolean { export function setFlags(commander: Object) { commander.description('Yarn install is used to install all dependencies for a project.'); commander.usage('install [flags]'); + commander.option('-A, --audit', 'Run vulnerability audit on installed packages'); commander.option('-g, --global', 'DEPRECATED'); commander.option('-S, --save', 'DEPRECATED - save package to your `dependencies`'); commander.option('-D, --save-dev', 'DEPRECATED - save package to your `devDependencies`'); diff --git a/src/cli/commands/upgrade.js b/src/cli/commands/upgrade.js index 9598b4a7ed..160c552d04 100644 --- a/src/cli/commands/upgrade.js +++ b/src/cli/commands/upgrade.js @@ -158,6 +158,7 @@ export function setFlags(commander: Object) { '-C, --caret', 'install most recent release with the same major version. Only used when --latest is specified.', ); + commander.option('-A', '--audit', 'Run vulnerability audit on installed packages'); } export function hasWrapper(commander: Object, args: Array): boolean { diff --git a/src/hoisted-tree-builder.js b/src/hoisted-tree-builder.js new file mode 100644 index 0000000000..e8f1b2c50a --- /dev/null +++ b/src/hoisted-tree-builder.js @@ -0,0 +1,86 @@ +/* @flow */ + +import type PackageResolver from './package-resolver.js'; +import type PackageLinker from './package-linker.js'; +import type {HoistManifest} from './package-hoister.js'; + +const invariant = require('invariant'); + +export type HoistedTree = { + name: string, + version: string, + manifest: HoistManifest, + children?: HoistedTrees, +}; +export type HoistedTrees = Array; + +export function getParent(key: string, treesByKey: Object): Object { + const parentKey = key.split('#').slice(0, -1).join('#'); + return treesByKey[parentKey]; +} + +export async function buildTree( + resolver: PackageResolver, + linker: PackageLinker, + patterns: Array, + ignoreHoisted?: boolean, +): Promise { + const treesByKey = {}; + const trees = []; + const flatTree = await linker.getFlatHoistedTree(patterns); + + // If using workspaces, filter out the virtual manifest + const {workspaceLayout} = resolver; + const hoisted = + workspaceLayout && workspaceLayout.virtualManifestName + ? flatTree.filter(([key]) => key.indexOf(workspaceLayout.virtualManifestName) === -1) + : flatTree; + + const hoistedByKey = {}; + for (const [key, info] of hoisted) { + hoistedByKey[key] = info; + } + + // build initial trees + for (const [, info] of hoisted) { + const ref = info.pkg._reference; + // const parent = getParent(info.key, treesByKey); + const children = []; + // let depth = 0; + invariant(ref, 'expected reference'); + + // check parent to obtain next depth + // if (parent && parent.depth > 0) { + // depth = parent.depth + 1; + // } else { + // depth = 0; + // } + + treesByKey[info.key] = { + name: info.pkg.name, + version: info.pkg.version, + children, + manifest: info, + }; + } + + // add children + for (const [, info] of hoisted) { + const tree = treesByKey[info.key]; + const parent = getParent(info.key, treesByKey); + if (!tree) { + continue; + } + + if (info.key.split('#').length === 1) { + trees.push(tree); + continue; + } + + if (parent) { + parent.children.push(tree); + } + } + + return trees; +} diff --git a/src/reporters/base-reporter.js b/src/reporters/base-reporter.js index 53dc5c175d..44653602b0 100644 --- a/src/reporters/base-reporter.js +++ b/src/reporters/base-reporter.js @@ -14,6 +14,8 @@ import type { } from './types.js'; import type {LanguageKeys} from './lang/en.js'; import type {Formatter} from './format.js'; +import type {AuditMetadata, AuditActionRecommendation, AuditAdvisory, AuditResolution} from '../cli/commands/audit'; + import {defaultFormatter} from './format.js'; import * as languages from './lang/index.js'; import isCI from 'is-ci'; @@ -223,9 +225,21 @@ export default class BaseReporter { // the screen shown at the very end of the CLI footer(showPeakMemory: boolean) {} - // + // a table structure table(head: Array, body: Array>) {} + // security audit action to resolve advisories + auditAction(recommendation: AuditActionRecommendation) {} + + // security audit requires manual review + auditManualReview() {} + + // security audit advisory + auditAdvisory(resolution: AuditResolution, auditAdvisory: AuditAdvisory) {} + + // summary for security audit report + auditSummary(auditMetadata: AuditMetadata) {} + // render an activity spinner and return a function that will trigger an update activity(): ReporterSpinner { return { diff --git a/src/reporters/console/console-reporter.js b/src/reporters/console/console-reporter.js index 6236511c0d..0fe162788c 100644 --- a/src/reporters/console/console-reporter.js +++ b/src/reporters/console/console-reporter.js @@ -11,6 +11,8 @@ import type { PromptOptions, } from '../types.js'; import type {FormatKeys} from '../format.js'; +import type {AuditMetadata, AuditActionRecommendation, AuditAdvisory, AuditResolution} from '../../cli/commands/audit'; + import BaseReporter from '../base-reporter.js'; import Progress from './progress-bar.js'; import Spinner from './spinner-progress.js'; @@ -18,6 +20,7 @@ import {clearLine} from './util.js'; import {removeSuffix} from '../../util/misc.js'; import {sortTrees, recurseTree, getFormattedOutput} from './helpers/tree-helper.js'; import inquirer from 'inquirer'; +import Table from 'cli-table3'; const {inspect} = require('util'); const readline = require('readline'); @@ -26,6 +29,16 @@ const stripAnsi = require('strip-ansi'); const read = require('read'); const tty = require('tty'); +const AUDIT_COL_WIDTHS = [15, 62]; + +const auditSeverityColors = { + info: chalk.bold, + low: chalk.bold, + moderate: chalk.yellow, + high: chalk.red, + critical: chalk.bgRed, +}; + type Row = Array; type InquirerResponses = {[key: K]: Array}; @@ -466,4 +479,106 @@ export default class ConsoleReporter extends BaseReporter { return answers[name]; } + + auditSummary(auditMetadata: AuditMetadata) { + const {totalDependencies, vulnerabilities} = auditMetadata; + const totalVulnerabilities = + vulnerabilities.info + + vulnerabilities.low + + vulnerabilities.moderate + + vulnerabilities.high + + vulnerabilities.critical; + const summary = this.lang( + 'auditSummary', + totalVulnerabilities > 0 ? this.rawText(chalk.red(totalVulnerabilities.toString())) : totalVulnerabilities, + totalDependencies, + ); + this._log(summary); + + if (totalVulnerabilities) { + const severities = []; + if (vulnerabilities.info) { + severities.push(this.lang('auditInfo', vulnerabilities.info)); + } + if (vulnerabilities.low) { + severities.push(this.lang('auditLow', vulnerabilities.low)); + } + if (vulnerabilities.moderate) { + severities.push(this.lang('auditModerate', vulnerabilities.moderate)); + } + if (vulnerabilities.high) { + severities.push(this.lang('auditHigh', vulnerabilities.high)); + } + if (vulnerabilities.critical) { + severities.push(this.lang('auditCritical', vulnerabilities.critical)); + } + this._log(`${this.lang('auditSummarySeverity')} ${severities.join(' | ')}`); + } + } + + auditAction(recommendation: AuditActionRecommendation) { + const label = recommendation.action.resolves.length === 1 ? 'vulnerability' : 'vulnerabilities'; + this._log( + this.lang( + 'auditResolveCommand', + this.rawText(chalk.inverse(recommendation.cmd)), + recommendation.action.resolves.length, + this.rawText(label), + ), + ); + if (recommendation.isBreaking) { + this._log(this.lang('auditSemverMajorChange')); + } + } + + auditManualReview() { + const tableOptions = { + colWidths: [78], + }; + const table = new Table(tableOptions); + table.push([ + { + content: this.lang('auditManualReview'), + vAlign: 'center', + hAlign: 'center', + }, + ]); + + this._log(table.toString()); + } + + auditAdvisory(resolution: AuditResolution, auditAdvisory: AuditAdvisory) { + function colorSeverity(severity: string, message: ?string): string { + return auditSeverityColors[severity](message || severity); + } + + function makeAdvisoryTableRow(patchedIn: ?string): Array { + const patchRows = []; + + if (patchedIn) { + patchRows.push({'Patched in': patchedIn}); + } + + return [ + {[chalk.bold(colorSeverity(auditAdvisory.severity))]: chalk.bold(auditAdvisory.title)}, + {Package: auditAdvisory.module_name}, + ...patchRows, + {'Dependency of': `${resolution.path.split('>')[0]} ${resolution.dev ? '[dev]' : ''}`}, + {Path: resolution.path.split('>').join(' > ')}, + {'More info': `https://nodesecurity.io/advisories/${auditAdvisory.id}`}, + ]; + } + + const tableOptions = { + colWidths: AUDIT_COL_WIDTHS, + wordWrap: true, + }; + const table = new Table(tableOptions); + const patchedIn = + auditAdvisory.patched_versions.replace(' ', '') === '<0.0.0' + ? 'No patch available' + : auditAdvisory.patched_versions; + table.push(...makeAdvisoryTableRow(patchedIn)); + this._log(table.toString()); + } } diff --git a/src/reporters/json-reporter.js b/src/reporters/json-reporter.js index df44f7f994..ecb7285910 100644 --- a/src/reporters/json-reporter.js +++ b/src/reporters/json-reporter.js @@ -1,6 +1,7 @@ /* @flow */ import type {ReporterSpinnerSet, Trees, ReporterSpinner} from './types.js'; +import type {AuditMetadata, AuditActionRecommendation, AuditAdvisory, AuditResolution} from '../cli/commands/audit'; import BaseReporter from './base-reporter.js'; export default class JSONReporter extends BaseReporter { @@ -160,4 +161,16 @@ export default class JSONReporter extends BaseReporter { } }; } + + auditAction(recommendation: AuditActionRecommendation) { + this._dump('auditAction', recommendation); + } + + auditAdvisory(resolution: AuditResolution, auditAdvisory: AuditAdvisory) { + this._dump('auditAdvisory', {resolution, advisory: auditAdvisory}); + } + + auditSummary(auditMetadata: AuditMetadata) { + this._dump('auditSummary', auditMetadata); + } } diff --git a/src/reporters/lang/en.js b/src/reporters/lang/en.js index 1f0468c23b..f5ee385e77 100644 --- a/src/reporters/lang/en.js +++ b/src/reporters/lang/en.js @@ -407,6 +407,21 @@ const messages = { verboseUpgradeUnlocking: 'Unlocking $0 in the lockfile.', folderMissing: "Directory $0 doesn't exist", mutexPortBusy: 'Cannot use the network mutex on port $0. It is probably used by another app.', + + auditRunning: 'Auditing packages', + auditSummary: '$0 vulnerabilities found - Packages audited: $1', + auditSummarySeverity: 'Severity:', + auditCritical: '$0 Critical', + auditHigh: '$0 High', + auditModerate: '$0 Moderate', + auditLow: '$0 Low', + auditInfo: '$0 Info', + auditResolveCommand: '# Run $0 to resolve $1 $2', + auditSemverMajorChange: 'SEMVER WARNING: Recommended action is a potentially breaking change', + auditManualReview: + 'Manual Review\nSome vulnerabilities require your attention to resolve\n\nVisit https://go.npm.me/audit-guide for additional guidance', + auditRunAuditForDetails: 'Security audit found potential problems. Run "yarn audit" for additional details.', + auditOffline: 'Skipping audit. Security audit cannot be performed in offline mode.', }; export type LanguageKeys = $Keys; diff --git a/src/util/hooks.js b/src/util/hooks.js index 4417b37f5c..90cd57b4a2 100644 --- a/src/util/hooks.js +++ b/src/util/hooks.js @@ -1,6 +1,6 @@ /* @flow */ -export type YarnHook = 'resolveStep' | 'fetchStep' | 'linkStep' | 'buildStep' | 'pnpStep'; +export type YarnHook = 'resolveStep' | 'fetchStep' | 'linkStep' | 'buildStep' | 'pnpStep' | 'auditStep'; const YARN_HOOKS_KEY = 'experimentalYarnHooks'; diff --git a/yarn.lock b/yarn.lock index 6809cdb40b..8dc0b8bf37 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1766,6 +1766,16 @@ cli-cursor@^2.1.0: dependencies: restore-cursor "^2.0.0" +cli-table3@^0.5.1: + version "0.5.1" + resolved "https://registry.yarnpkg.com/cli-table3/-/cli-table3-0.5.1.tgz#0252372d94dfc40dbd8df06005f48f31f656f202" + integrity sha512-7Qg2Jrep1S/+Q3EceiZtQcDPWxhAvBw+ERf1162v4sikJrvojMHFqXt8QIVha8UlH9rgU0BeWPytZ9/TzYqlUw== + dependencies: + object-assign "^4.1.0" + string-width "^2.1.1" + optionalDependencies: + colors "^1.1.2" + cli-width@^2.0.0: version "2.2.0" resolved "https://registry.yarnpkg.com/cli-width/-/cli-width-2.2.0.tgz#ff19ede8a9a5e579324147b0c11f0fbcbabed639" @@ -1866,6 +1876,11 @@ color-support@^1.1.3: resolved "https://registry.yarnpkg.com/color-support/-/color-support-1.1.3.tgz#93834379a1cc9a0c61f82f52f0d04322251bd5a2" integrity sha512-qiBjkpbMLO/HL68y+lh4q0/O1MZFj2RX6X/KmMa3+gJD3z+WwI1ZzDHysvqHGS3mP6mznPckpXmw1nI9cJjyRg== +colors@^1.1.2: + version "1.3.2" + resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.2.tgz#2df8ff573dfbf255af562f8ce7181d6b971a359b" + integrity sha512-rhP0JSBGYvpcNQj4s5AdShMeE5ahMop96cTeDl/v9qQQm2fYClE2QXZRi8wLzc+GmXSxdIqqbOIAhyObEXDbfQ== + combined-stream@1.0.6, combined-stream@~1.0.5: version "1.0.6" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.6.tgz#723e7df6e801ac5613113a7e445a9b69cb632818"