From b62aab571e1bd7135485dce91bfdac90aa15b4d3 Mon Sep 17 00:00:00 2001 From: Jahed Ahmed Date: Thu, 29 Jul 2021 17:54:37 +0000 Subject: [PATCH] test: migrate fail-on to jest --- test/acceptance/cli-fail-on.test.ts | 442 ------------------ test/acceptance/fake-server.ts | 3 +- .../jest/acceptance/snyk-test/fail-on.spec.ts | 52 +++ test/jest/util/createProject.ts | 40 +- 4 files changed, 91 insertions(+), 446 deletions(-) delete mode 100644 test/acceptance/cli-fail-on.test.ts create mode 100644 test/jest/acceptance/snyk-test/fail-on.spec.ts diff --git a/test/acceptance/cli-fail-on.test.ts b/test/acceptance/cli-fail-on.test.ts deleted file mode 100644 index f18c2c3520d..00000000000 --- a/test/acceptance/cli-fail-on.test.ts +++ /dev/null @@ -1,442 +0,0 @@ -import * as tap from 'tap'; -import * as cli from '../../src/cli/commands'; -import { fakeServer } from './fake-server'; -import { chdirWorkspaces, getWorkspaceJSON } from './workspace-helper'; - -const { test, only } = tap; -(tap as any).runOnly = false; // <- for debug. set to true, and replace a test to only(..) - -const port = (process.env.PORT = process.env.SNYK_PORT = '12345'); -const BASE_API = '/api/v1'; -process.env.SNYK_API = 'http://localhost:' + port + BASE_API; -process.env.SNYK_HOST = 'http://localhost:' + port; -process.env.LOG_LEVEL = '0'; -const apiKey = '123456789'; -let oldkey; -let oldendpoint; -const server = fakeServer(BASE_API, apiKey); -const before = tap.runOnly ? only : test; -const after = tap.runOnly ? only : test; - -// fake server responses -const noVulnsResult = getWorkspaceJSON( - 'fail-on', - 'no-vulns', - 'vulns-result.json', -); -const noFixableResult = getWorkspaceJSON( - 'fail-on', - 'no-fixable', - 'vulns-result.json', -); -const upgradableResult = getWorkspaceJSON( - 'fail-on', - 'upgradable', - 'vulns-result.json', -); -const patchableResult = getWorkspaceJSON( - 'fail-on', - 'patchable', - 'vulns-result.json', -); - -// @later: remove this config stuff. -// Was copied straight from ../src/cli-server.js -before('setup', async (t) => { - t.plan(3); - let key = await cli.config('get', 'api'); - oldkey = key; - t.pass('existing user config captured'); - - key = await cli.config('get', 'endpoint'); - oldendpoint = key; - t.pass('existing user endpoint captured'); - - await new Promise((resolve) => { - server.listen(port, resolve); - }); - t.pass('started demo server'); - t.end(); -}); - -// @later: remove this config stuff. -// Was copied straight from ../src/cli-server.js -before('prime config', async (t) => { - await cli.config('set', 'api=' + apiKey); - t.pass('api token set'); - await cli.config('unset', 'endpoint'); - t.pass('endpoint removed'); - t.end(); -}); - -// --fail-on=all -test('test project with no vulns and --fail-on=all', async (t) => { - try { - server.setNextResponse(noVulnsResult); - chdirWorkspaces('fail-on'); - await cli.test('no-vulns', { - failOn: 'all', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with no fixable and --fail-on=all', async (t) => { - try { - server.setNextResponse(noFixableResult); - chdirWorkspaces('fail-on'); - await cli.test('no-fixable', { - failOn: 'all', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with upgradable and --fail-on=all', async (t) => { - try { - server.setNextResponse(upgradableResult); - chdirWorkspaces('fail-on'); - await cli.test('upgradable', { - failOn: 'all', - }); - t.fail('expected exception to be thrown'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with patchable and --fail-on=all', async (t) => { - try { - server.setNextResponse(patchableResult); - chdirWorkspaces('fail-on'); - await cli.test('patchable', { - failOn: 'all', - }); - t.fail('expected exception to be thrown'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with upgradable and --fail-on=all --json', async (t) => { - try { - server.setNextResponse(upgradableResult); - chdirWorkspaces('fail-on'); - await cli.test('upgradable', { - failOn: 'all', - json: true, - }); - t.fail('expected exception to be thrown'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with patchable and --fail-on=all --json', async (t) => { - try { - server.setNextResponse(patchableResult); - chdirWorkspaces('fail-on'); - await cli.test('patchable', { - failOn: 'all', - json: true, - }); - t.fail('expected exception to be thrown'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with no fixable and --fail-on=all --json', async (t) => { - try { - server.setNextResponse(noFixableResult); - chdirWorkspaces('fail-on'); - await cli.test('no-fixable', { - failOn: 'all', - json: true, - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test project with no vulns and --fail-on=all --json', async (t) => { - try { - server.setNextResponse(noVulnsResult); - chdirWorkspaces('fail-on'); - await cli.test('no-vulns', { - failOn: 'all', - json: true, - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -// --fail-on=upgradable -test('test project with no vulns and --fail-on=upgradable', async (t) => { - try { - server.setNextResponse(noVulnsResult); - chdirWorkspaces('fail-on'); - await cli.test('no-vulns', { - failOn: 'upgradable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with no fixable and --fail-on=upgradable', async (t) => { - try { - server.setNextResponse(noFixableResult); - chdirWorkspaces('fail-on'); - await cli.test('no-fixable', { - failOn: 'upgradable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with upgradable and --fail-on=upgradable', async (t) => { - try { - server.setNextResponse(upgradableResult); - chdirWorkspaces('fail-on'); - await cli.test('upgradable', { - failOn: 'upgradable', - }); - t.fail('expected test to throw exception'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); -test('test vulnerable project with patchable and --fail-on=upgradable', async (t) => { - try { - server.setNextResponse(patchableResult); - chdirWorkspaces('fail-on'); - await cli.test('patchable', { - failOn: 'upgradable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with upgradable and --fail-on=upgradable --json', async (t) => { - try { - server.setNextResponse(upgradableResult); - chdirWorkspaces('fail-on'); - await cli.test('upgradable', { - failOn: 'upgradable', - json: true, - }); - t.fail('expected test to throw exception'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with patchable and --fail-on=upgradable --json', async (t) => { - try { - server.setNextResponse(patchableResult); - chdirWorkspaces('fail-on'); - await cli.test('patchable', { - failOn: 'upgradable', - json: true, - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with no fixable and --fail-on=upgradable --json', async (t) => { - try { - server.setNextResponse(noFixableResult); - chdirWorkspaces('fail-on'); - await cli.test('no-fixable', { - failOn: 'upgradable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test project with no vulns and --fail-on=upgradable --json', async (t) => { - try { - server.setNextResponse(noVulnsResult); - chdirWorkspaces('fail-on'); - await cli.test('no-vulns', { - failOn: 'upgradable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -// --fail-on=patchable -test('test project with no vulns and --fail-on=patchable', async (t) => { - try { - server.setNextResponse(noVulnsResult); - chdirWorkspaces('fail-on'); - await cli.test('no-vulns', { - failOn: 'patchable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with no fixable and --fail-on=patchable', async (t) => { - try { - server.setNextResponse(noVulnsResult); - chdirWorkspaces('fail-on'); - await cli.test('no-fixable', { - failOn: 'patchable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with patchable and --fail-on=patchable', async (t) => { - try { - server.setNextResponse(patchableResult); - chdirWorkspaces('fail-on'); - await cli.test('patchable', { - failOn: 'patchable', - }); - t.fail('expected test to throw exception'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with upgradable and --fail-on=patchable', async (t) => { - try { - server.setNextResponse(upgradableResult); - chdirWorkspaces('fail-on'); - await cli.test('upgradable', { - failOn: 'patchable', - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test vulnerable project with patchable and --fail-on=patchable --json', async (t) => { - try { - server.setNextResponse(patchableResult); - chdirWorkspaces('fail-on'); - await cli.test('patchable', { - failOn: 'patchable', - json: true, - }); - t.fail('expected test to throw exception'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with upgradable and --fail-on=patchable --json', async (t) => { - try { - server.setNextResponse(patchableResult); - chdirWorkspaces('fail-on'); - await cli.test('patchable', { - failOn: 'patchable', - json: true, - }); - t.fail('expected test to throw exception'); - } catch (err) { - t.equal(err.code, 'VULNS', 'should throw exception'); - } -}); - -test('test vulnerable project with no fixable and --fail-on=patchable --json', async (t) => { - try { - server.setNextResponse(noFixableResult); - chdirWorkspaces('fail-on'); - await cli.test('no-fixable', { - failOn: 'patchable', - json: true, - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -test('test project with no vulns and --fail-on=patchable --json', async (t) => { - try { - server.setNextResponse(noVulnsResult); - chdirWorkspaces('fail-on'); - await cli.test('no-vulns', { - failOn: 'patchable', - json: true, - }); - t.pass('should not throw exception'); - } catch (err) { - t.fail('did not expect exception to be thrown ' + err); - } -}); - -// test invalid arg -test('test project with --fail-on=invalid', async (t) => { - try { - chdirWorkspaces(); - await cli.test('npm-package', { - failOn: 'invalid', - }); - t.fail('expected invalid fail-on to throw exception'); - } catch (err) { - t.equal( - err, - 'Invalid fail on argument, please use one of: all | upgradable | patchable', - ); - } -}); - -// @later: try and remove this config stuff -// Was copied straight from ../src/cli-server.js -after('teardown', async (t) => { - t.plan(4); - - delete process.env.SNYK_API; - delete process.env.SNYK_HOST; - delete process.env.SNYK_PORT; - t.notOk(process.env.SNYK_PORT, 'fake env values cleared'); - - await new Promise((resolve) => { - server.close(resolve); - }); - t.pass('server shutdown'); - let key = 'set'; - let value = 'api=' + oldkey; - if (!oldkey) { - key = 'unset'; - value = 'api'; - } - await cli.config(key, value); - t.pass('user config restored'); - if (oldendpoint) { - await cli.config('endpoint', oldendpoint); - t.pass('user endpoint restored'); - t.end(); - } else { - t.pass('no endpoint'); - t.end(); - } -}); diff --git a/test/acceptance/fake-server.ts b/test/acceptance/fake-server.ts index cdbee183385..cec01673c1e 100644 --- a/test/acceptance/fake-server.ts +++ b/test/acceptance/fake-server.ts @@ -345,7 +345,8 @@ export function fakeServer(root, apikey) { }); server.setNextResponse = (response) => { - server._nextResponse = response; + server._nextResponse = + typeof response === 'string' ? JSON.parse(response) : response; }; server.setNextStatusCodeAndResponse = (code, body) => { diff --git a/test/jest/acceptance/snyk-test/fail-on.spec.ts b/test/jest/acceptance/snyk-test/fail-on.spec.ts new file mode 100644 index 00000000000..3a4486e0908 --- /dev/null +++ b/test/jest/acceptance/snyk-test/fail-on.spec.ts @@ -0,0 +1,52 @@ +import { createProjectFromWorkspace } from '../../util/createProject'; +import { runSnykCLI } from '../../util/runSnykCLI'; +import { fakeServer } from '../../../acceptance/fake-server'; + +jest.setTimeout(1000 * 60); + +describe('snyk test --fail-on', () => { + let server: ReturnType; + let env: Record; + + beforeAll((done) => { + const apiPath = '/api/v1'; + const apiPort = process.env.PORT || process.env.SNYK_PORT || '12345'; + env = { + ...process.env, + SNYK_API: 'http://localhost:' + apiPort + apiPath, + SNYK_TOKEN: '123456789', + }; + + server = fakeServer(apiPath, env.SNYK_TOKEN); + server.listen(apiPort, () => done()); + }); + + afterAll((done) => { + server.close(() => done()); + }); + + test.each([ + ['all', 'no-vulns', 0], + ['all', 'no-fixable', 0], + ['all', 'upgradable', 1], + ['all', 'patchable', 1], + ['upgradable', 'upgradable', 1], + ['upgradable', 'patchable', 0], + ['patchable', 'upgradable', 0], + ['patchable', 'patchable', 1], + ['invalid-value', 'no-vulns', 2], + ])( + 'snyk test --fail-on="%s" with %s result exits with %i', + async (failOn, workspace, expectedCode) => { + const project = await createProjectFromWorkspace('fail-on/' + workspace); + server.setNextResponse(await project.read('vulns-result.json')); + + const { code } = await runSnykCLI(`test --fail-on=${failOn}`, { + cwd: project.path(), + env, + }); + + expect(code).toEqual(expectedCode); + }, + ); +}); diff --git a/test/jest/util/createProject.ts b/test/jest/util/createProject.ts index ee9c9c9fdd4..25f33b5cd4f 100644 --- a/test/jest/util/createProject.ts +++ b/test/jest/util/createProject.ts @@ -8,7 +8,14 @@ type TestProject = { remove: () => Promise; }; -const createProject = async (fixtureName: string): Promise => { +/** + * Copies a fixture to a temporary directory so that tests can be isolated + * with minimal overlap and cleanup. + */ +const createProject = async ( + fixtureName: string, + fixturePath: string, +): Promise => { const tempFolder = await fse.promises.mkdtemp( path.resolve( os.tmpdir(), @@ -16,7 +23,6 @@ const createProject = async (fixtureName: string): Promise => { ), ); - const fixturePath = path.resolve(__dirname, '../../fixtures', fixtureName); const projectPath = path.resolve(tempFolder, fixtureName); await fse.copy(fixturePath, projectPath); @@ -32,4 +38,32 @@ const createProject = async (fixtureName: string): Promise => { }; }; -export { createProject }; +/** + * Workaround until we move all fixtures to ./test/fixtures + */ +const createProjectFromWorkspace = async ( + fixtureName: string, +): Promise => { + return createProject( + fixtureName, + path.join(__dirname, '../../acceptance/workspaces/' + fixtureName), + ); +}; + +/** + * Once createProjectFromWorkspace is removed, this can be "createProject". + */ +const createProjectFromFixture = async ( + fixtureName: string, +): Promise => { + return createProject( + fixtureName, + path.join(__dirname, '../../fixtures', fixtureName), + ); +}; + +export { + createProjectFromFixture as createProject, + createProjectFromFixture, + createProjectFromWorkspace, +};