feat: test dependencies for ecosystems

Test the dependencies of each ecosystem scan result.

package.json

@@ -30,7 +30,7 @@
     "test:system": "tap test/system/*.test.* -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register",
     "test:jest": "jest test/*.spec.ts",
     "test:test": "tap test/*.test.* -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register",
-    "test": "npm run test:acceptance && npm run test:system && npm run test:test && npm run test:jest",
+    "test": "npm run test:acceptance && npm run test:system && npm run test:jest",
     "test-windows": "tap test/acceptance/**/*.test.* -Rspec --timeout=300 --node-arg=-r --node-arg=ts-node/register && npm run test:system && npm run test:test",
     "lint": "run-p --aggregate-output lint:*",
     "lint:js": "eslint --color --cache 'src/**/*.{js,ts}'",
@@ -76,7 +76,7 @@
     "proxy-from-env": "^1.0.0",
     "semver": "^6.0.0",
     "snyk-config": "3.1.0",
-    "snyk-cpp-plugin": "1.2.0",
+    "snyk-cpp-plugin": "1.4.0",
     "snyk-docker-plugin": "3.17.0",
     "snyk-go-plugin": "1.16.0",
     "snyk-gradle-plugin": "3.5.1",

src/cli/commands/test/index.ts

@@ -109,12 +109,16 @@ async function test(...args: MethodArgs): Promise<TestCommandResult> {
 
   const ecosystem = getEcosystem(options);
   if (ecosystem) {
-    const commandResult = await testEcosystem(
-      ecosystem,
-      args as string[],
-      options,
-    );
-    return commandResult;
+    try {
+      const commandResult = await testEcosystem(
+        ecosystem,
+        args as string[],
+        options,
+      );
+      return commandResult;
+    } catch (error) {
+      throw new Error(error);
+    }
   }
 
   // Promise waterfall to test all other paths sequentially
@@ -141,6 +145,7 @@ async function test(...args: MethodArgs): Promise<TestCommandResult> {
       //
       // To standardise this, make sure we use the best _object_ to
       // describe the error.
+
       if (error instanceof Error) {
         res = error;
       } else if (typeof error !== 'object') {

src/lib/ecosystems.ts

@@ -1,20 +1,57 @@
 import * as cppPlugin from 'snyk-cpp-plugin';
 import { Options } from './types';
 import { TestCommandResult } from '../cli/commands/types';
+import * as config from './config';
+import { isCI } from './is-ci';
+import * as snyk from './';
+import request = require('./request');
+import { DepGraphData } from '@snyk/dep-graph';
 
 interface Artifact {
   type: string;
   data: any;
-  meta?: { [key: string]: any };
+  meta: { [key: string]: any };
 }
 
 interface ScanResult {
+  type: string;
   artifacts: Artifact[];
+  meta: {
+    [key: string]: any;
+  };
+}
+
+interface TestResults {
+  depGraph: DepGraphData;
+  affectedPkgs: {
+    [pkgId: string]: {
+      pkg: {
+        name: string;
+        version: string;
+      };
+      issues: {
+        [issueId: string]: {
+          issueId: string;
+        };
+      };
+    };
+  };
+  issuesData: {
+    [issueId: string]: {
+      id: string;
+      severity: string;
+      title: string;
+    };
+  };
 }
 
 export interface EcosystemPlugin {
   scan: (options: Options) => Promise<ScanResult[]>;
-  display: (scanResults: ScanResult[]) => Promise<string>;
+  display: (
+    scanResults: ScanResult[],
+    testResults: TestResults[],
+    errors: string[],
+  ) => Promise<string>;
 }
 
 export type Ecosystem = 'cpp';
@@ -42,21 +79,76 @@ export async function testEcosystem(
   options: Options,
 ): Promise<TestCommandResult> {
   const plugin = getPlugin(ecosystem);
-  let allScanResults: ScanResult[] = [];
+  const scanResultsByPath: { [dir: string]: ScanResult[] } = {};
+  let scanResults: ScanResult[] = [];
   for (const path of paths) {
     options.path = path;
-    const scanResults = await plugin.scan(options);
-    allScanResults = allScanResults.concat(scanResults);
+    const results = await plugin.scan(options);
+    scanResultsByPath[path] = results;
+    scanResults = scanResults.concat(results);
   }
 
-  const stringifiedData = JSON.stringify(allScanResults, null, 2);
+  const [testResults, errors] = await testDependencies(scanResultsByPath);
+  const stringifiedData = JSON.stringify(testResults, null, 2);
   if (options.json) {
     return TestCommandResult.createJsonTestCommandResult(stringifiedData);
   }
+  const readableResult = await plugin.display(scanResults, testResults, errors);
 
-  const readableResult = await plugin.display(allScanResults);
   return TestCommandResult.createHumanReadableTestCommandResult(
     readableResult,
     stringifiedData,
   );
 }
+
+export async function testDependencies(scans: {
+  [dir: string]: ScanResult[];
+}): Promise<[TestResults[], string[]]> {
+  const results: TestResults[] = [];
+  const errors: string[] = [];
+  for (const [path, scanResults] of Object.entries(scans)) {
+    for (const scanResult of scanResults) {
+      const payload = {
+        method: 'POST',
+        url: `${config.API}/test-dependencies`,
+        json: true,
+        headers: {
+          'x-is-ci': isCI(),
+          authorization: 'token ' + snyk.api,
+        },
+        body: {
+          type: 'cpp',
+          artifacts: scanResult.artifacts,
+          meta: {},
+        },
+      };
+      try {
+        const response = await makeRequest(payload);
+        results.push(response);
+      } catch (error) {
+        if (error.code !== 200) {
+          throw new Error(error.message);
+        }
+        errors.push('Could not test dependencies in ' + path);
+      }
+    }
+  }
+  return [results, errors];
+}
+
+export async function makeRequest(payload: any): Promise<TestResults> {
+  return new Promise((resolve, reject) => {
+    request(payload, (error, res, body) => {
+      if (error) {
+        return reject(error);
+      }
+      if (res.statusCode !== 200) {
+        return reject({
+          code: res.statusCode,
+          message: res?.body?.message || 'Error testing dependencies',
+        });
+      }
+      resolve(body);
+    });
+  });
+}

test/ecosystems.spec.ts

@@ -2,19 +2,19 @@ import { Options } from '../src/lib/types';
 import * as cppPlugin from 'snyk-cpp-plugin';
 import * as path from 'path';
 import * as fs from 'fs';
-import { getPlugin, getEcosystem, testEcosystem } from '../src/lib/ecosystems';
+import * as ecosystems from '../src/lib/ecosystems';
 import { TestCommandResult } from '../src/cli/commands/types';
 
 describe('ecosystems', () => {
   describe('getPlugin', () => {
     it('should return c++ plugin when cpp ecosystem is given', () => {
-      const actual = getPlugin('cpp');
+      const actual = ecosystems.getPlugin('cpp');
       const expected = cppPlugin;
       expect(actual).toBe(expected);
     });
 
     it('should return undefined when ecosystem is not supported', () => {
-      const actual = getPlugin('unsupportedEcosystem' as any);
+      const actual = ecosystems.getPlugin('unsupportedEcosystem' as any);
       const expected = undefined;
       expect(actual).toBe(expected);
     });
@@ -26,61 +26,95 @@ describe('ecosystems', () => {
         source: true,
         path: '',
       };
-      const actual = getEcosystem(options);
+      const actual = ecosystems.getEcosystem(options);
       const expected = 'cpp';
       expect(actual).toBe(expected);
     });
+  });
 
-    it('should return null when options source is false', () => {
-      const options: Options = {
-        source: false,
-        path: '',
-      };
-      const actual = getEcosystem(options);
-      const expected = null;
-      expect(actual).toBe(expected);
-    });
+  it('should return null when options source is false', () => {
+    const options: Options = {
+      source: false,
+      path: '',
+    };
+    const actual = ecosystems.getEcosystem(options);
+    const expected = null;
+    expect(actual).toBe(expected);
   });
+});
 
-  describe('testEcosystem', () => {
-    const fixturePath = path.join(__dirname, 'fixtures', 'cpp-project');
-    const cwd = process.cwd();
+describe('testEcosystem', () => {
+  const fixturePath = path.join(__dirname, 'fixtures', 'cpp-project');
+  const cwd = process.cwd();
 
-    function readFixture(filename: string) {
-      const filePath = path.join(fixturePath, filename);
-      return fs.readFileSync(filePath, 'utf-8');
-    }
+  function readFixture(filename: string) {
+    const filePath = path.join(fixturePath, filename);
+    return fs.readFileSync(filePath, 'utf-8');
+  }
 
-    beforeAll(() => {
-      process.chdir(fixturePath);
-    });
-    afterAll(() => {
-      process.chdir(cwd);
-    });
+  beforeAll(() => {
+    process.chdir(fixturePath);
+  });
 
-    it('should return human readable result when no json option given', async () => {
-      const display = readFixture('display.txt');
-      const scan = readFixture('scan.json');
-      const stringifiedData = JSON.stringify(JSON.parse(scan), null, 2);
-      const expected = TestCommandResult.createHumanReadableTestCommandResult(
-        display,
-        stringifiedData,
-      );
-      const actual = await testEcosystem('cpp', ['.'], { path: '' });
-      expect(actual).toEqual(expected);
+  beforeEach(() => {
+    jest.resetAllMocks();
+  });
+
+  afterAll(() => {
+    process.chdir(cwd);
+  });
+
+  it('should return human readable result when no json option given', async () => {
+    const display = readFixture('display.txt');
+    const testResults = readFixture('testResults.json');
+    const stringifiedData = JSON.stringify(JSON.parse(testResults), null, 2);
+    const expected = TestCommandResult.createHumanReadableTestCommandResult(
+      display,
+      stringifiedData,
+    );
+
+    const actual = await ecosystems.testEcosystem('cpp', ['.'], { path: '' });
+    expect(actual).toEqual(expected);
+  });
+
+  it('should return json result when json option', async () => {
+    const testResults = readFixture('testResults.json');
+    const stringifiedData = JSON.stringify(JSON.parse(testResults), null, 2);
+    const expected = TestCommandResult.createJsonTestCommandResult(
+      stringifiedData,
+    );
+    const actual = await ecosystems.testEcosystem('cpp', ['.'], {
+      path: '',
+      json: true,
     });
+    expect(actual).toEqual(expected);
+  });
 
-    it('should return json result when json option', async () => {
-      const scan = readFixture('scan.json');
-      const stringifiedData = JSON.stringify(JSON.parse(scan), null, 2);
-      const expected = TestCommandResult.createJsonTestCommandResult(
-        stringifiedData,
-      );
-      const actual = await testEcosystem('cpp', ['.'], {
+  it('should throw error when response code is not 200', async () => {
+    const expected = { code: 401, message: 'Invalid auth token' };
+    jest.spyOn(ecosystems, 'testEcosystem').mockRejectedValue(expected);
+    expect.assertions(1);
+    try {
+      await ecosystems.testEcosystem('cpp', ['.'], {
         path: '',
-        json: true,
       });
-      expect(actual).toEqual(expected);
+    } catch (error) {
+      expect(error).toEqual(expected);
+    }
+  });
+
+  it.skip('should return error when there was a problem testing dependencies', async () => {
+    //@boost: TODO finish up my implementation
+    // const makeRequestSpy = jest
+    //   .spyOn(ecosystems, 'makeRequest')
+    //   .mockRejectedValue('Something went wrong');
+    // const ecosystemDisplaySpy = jest.spyOn(cppPlugin, 'display');
+    const commandResult = await ecosystems.testEcosystem('cpp', ['.'], {
+      path: '',
+      json: true,
     });
+    console.log(commandResult);
+    expect(commandResult).toEqual('');
+    // expect(ecosystemDisplaySpy).toHaveBeenCalledWith({});
   });
 });

test/fixtures/cpp-project/display.txt

@@ -1,3 +1,9 @@
+Dependency Fingerprints
+-----------------------
 52d1b046047db9ea0c581cafd4c68fe5 add.cpp
 aeca71a6e39f99a24ecf4c088eee9cb8 add.h
-ad3365b3370ef6b1c3e778f875055f19 main.cpp
+ad3365b3370ef6b1c3e778f875055f19 main.cpp
+
+Issues
+------
+Tested 0 dependencies for known issues, found 0 issues.