From 9151ff03bd0858516a8cf1ba95767d1ae9dd3a0d Mon Sep 17 00:00:00 2001 From: ghe Date: Wed, 2 Sep 2020 10:35:08 +0100 Subject: [PATCH 1/4] feat: exiut code 3 for no detected projects --- src/cli/index.ts | 8 ++++++++ src/lib/snyk-test/index.js | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/src/cli/index.ts b/src/cli/index.ts index 97077dc9e91..fac3922fe30 100755 --- a/src/cli/index.ts +++ b/src/cli/index.ts @@ -43,6 +43,7 @@ const debug = Debug('snyk'); const EXIT_CODES = { VULNS_FOUND: 1, ERROR: 2, + NO_SUPPORTED_MANIFESTS_FOUND: 3, }; async function runCommand(args: Args) { @@ -89,6 +90,13 @@ async function handleError(args, error) { spinner.clearAll(); let command = 'bad-command'; let exitCode = EXIT_CODES.ERROR; + const noSupportedManifestsFound = error.message?.includes( + 'Could not detect supported target files in', + ); + + if (noSupportedManifestsFound) { + exitCode = EXIT_CODES.NO_SUPPORTED_MANIFESTS_FOUND; + } const vulnsFound = error.code === 'VULNS'; if (vulnsFound) { diff --git a/src/lib/snyk-test/index.js b/src/lib/snyk-test/index.js index 307211de816..ae906bb710e 100644 --- a/src/lib/snyk-test/index.js +++ b/src/lib/snyk-test/index.js @@ -51,7 +51,9 @@ function executeTest(root, options) { return results; }); } catch (error) { - return Promise.reject(chalk.red.bold(error)); + return Promise.reject( + chalk.red.bold(error.message ? error.message : error), + ); } } From 809a73f43d6778da9966734789c13d6044199672 Mon Sep 17 00:00:00 2001 From: ghe Date: Thu, 3 Sep 2020 11:19:47 +0100 Subject: [PATCH 2/4] test: refactor & test exit codes --- test/acceptance/cli-args.test.ts | 109 +--------------------- test/fixtures/empty/not-supported.format | 0 test/smoke/README.md | 24 ++--- test/smoke/spec/snyk_test_spec.sh | 21 ++++- test/system/cli-json-file-output.test.ts | 114 +++++++++++++++++++++++ 5 files changed, 150 insertions(+), 118 deletions(-) create mode 100644 test/fixtures/empty/not-supported.format create mode 100644 test/system/cli-json-file-output.test.ts diff --git a/test/acceptance/cli-args.test.ts b/test/acceptance/cli-args.test.ts index 55743207e98..743595782fe 100644 --- a/test/acceptance/cli-args.test.ts +++ b/test/acceptance/cli-args.test.ts @@ -1,8 +1,6 @@ import { test } from 'tap'; import { exec } from 'child_process'; -import { sep, join } from 'path'; -import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs'; -import { v4 as uuidv4 } from 'uuid'; +import { sep } from 'path'; const osName = require('os-name'); @@ -101,7 +99,7 @@ test('snyk test command should fail when iac file is not supported', (t) => { } t.match( stdout.trim(), - 'CustomError: Illegal infrastructure as code target file', + 'Illegal infrastructure as code target file', 'correct error output', ); }, @@ -118,7 +116,7 @@ test('snyk test command should fail when iac file is not supported', (t) => { } t.match( stdout.trim(), - 'CustomError: Not supported infrastructure as code target files in', + 'Not supported infrastructure as code target files in', 'correct error output', ); }, @@ -346,104 +344,3 @@ test('`test --json-file-output no value produces error message`', (t) => { optionsToTest.forEach(validate); }); - -test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => { - t.plan(2); - - exec( - `node ${main} test --json-file-output=snyk-direct-json-test-output.json`, - (err, stdout) => { - if (err) { - throw err; - } - t.match(stdout, 'Organization:', 'contains human readable output'); - const outputFileContents = readFileSync( - 'snyk-direct-json-test-output.json', - 'utf-8', - ); - unlinkSync('./snyk-direct-json-test-output.json'); - const jsonObj = JSON.parse(outputFileContents); - const okValue = jsonObj.ok as boolean; - t.ok(okValue, 'JSON output ok'); - }, - ); -}); - -test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => { - t.plan(1); - - exec( - `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync( - 'snyk-direct-json-test-output.json', - 'utf-8', - ); - unlinkSync('./snyk-direct-json-test-output.json'); - t.equals(stdoutJson, outputFileContents); - }, - ); -}); - -test('`test --json-file-output can handle a relative path`', (t) => { - t.plan(1); - - // if 'test-output' doesn't exist, created it - if (!existsSync('test-output')) { - mkdirSync('test-output'); - } - - const tempFolder = uuidv4(); - const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`; - - exec( - `node ${main} test --json --json-file-output=${outputPath}`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync(outputPath, 'utf-8'); - unlinkSync(outputPath); - rmdirSync(`test-output/${tempFolder}`); - t.equals(stdoutJson, outputFileContents); - }, - ); -}); - -test( - '`test --json-file-output can handle an absolute path`', - { skip: iswindows }, - (t) => { - t.plan(1); - - // if 'test-output' doesn't exist, created it - if (!existsSync('test-output')) { - mkdirSync('test-output'); - } - - const tempFolder = uuidv4(); - const outputPath = join( - process.cwd(), - `test-output/${tempFolder}/snyk-direct-json-test-output.json`, - ); - - exec( - `node ${main} test --json --json-file-output=${outputPath}`, - (err, stdout) => { - if (err) { - throw err; - } - const stdoutJson = stdout; - const outputFileContents = readFileSync(outputPath, 'utf-8'); - unlinkSync(outputPath); - rmdirSync(`test-output/${tempFolder}`); - t.equals(stdoutJson, outputFileContents); - }, - ); - }, -); diff --git a/test/fixtures/empty/not-supported.format b/test/fixtures/empty/not-supported.format new file mode 100644 index 00000000000..e69de29bb2d diff --git a/test/smoke/README.md b/test/smoke/README.md index af89fe8d90c..9c30e7d54df 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -4,7 +4,9 @@ Design goal is to have a single test suite, that can detect if CLI is not workin CLI is being tested by a series of tests using [Shellspec](https://shellspec.info). See them in a `test/smoke/spec` folder. -Spec in this folder is used as a 1) **"Smoke test" step in CircleCI** to verify that built CLI can run 2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working. +Spec in this folder is used as a +1) **"Smoke test" step in CircleCI** to verify that built CLI can run +2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working. ## How to add a new smoke test @@ -14,18 +16,18 @@ Before you start adding specs, those files are bash scripts, it's recommended to It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5). -To run these tests locally, install: +To run these tests locally: -- [Shellspec](https://shellspec.info) -- [jq](https://stedolan.github.io/jq/) -- timeout (if not available on your platform) +1. Install: -cd into `test/smoke` folder and run: - -```sh -cd test/smoke -CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d -``` + - [Shellspec](https://shellspec.info) + - [jq](https://stedolan.github.io/jq/) + - timeout (if not available on your platform) +2. Run: + ```sh + cd test/smoke + CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d + ``` ## TODO diff --git a/test/smoke/spec/snyk_test_spec.sh b/test/smoke/spec/snyk_test_spec.sh index 5facc79dcb4..66f430caa13 100644 --- a/test/smoke/spec/snyk_test_spec.sh +++ b/test/smoke/spec/snyk_test_spec.sh @@ -10,9 +10,28 @@ Describe "Snyk test command" snyk test } + run_test_in_empty_subfolder() { + cd ../fixtures/empty || return + snyk test + } + + It "throws error when file does not exist" + When run snyk test --file=non-existent/package.json + The status should equal 2 + The output should include "Could not find the specified file" + The stderr should equal "" + End + + It "throws error when no suppored manifests detected" + When run run_test_in_empty_subfolder + The status should equal 3 + The output should include "Could not detect supported target files in" + The stderr should equal "" + End + It "finds vulns in a project in the same folder" When run run_test_in_subfolder - The status should be failure # issues found + The status should equal 1 The output should include "https://snyk.io/vuln/npm:minimatch:20160620" The stderr should equal "" End diff --git a/test/system/cli-json-file-output.test.ts b/test/system/cli-json-file-output.test.ts new file mode 100644 index 00000000000..36e0e356dc2 --- /dev/null +++ b/test/system/cli-json-file-output.test.ts @@ -0,0 +1,114 @@ +import { test } from 'tap'; +import { exec } from 'child_process'; +import { sep, join } from 'path'; +import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs'; +import { v4 as uuidv4 } from 'uuid'; + +const osName = require('os-name'); + +const main = './dist/cli/index.js'.replace(/\//g, sep); +const iswindows = + osName() + .toLowerCase() + .indexOf('windows') === 0; + +test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => { + t.plan(2); + + exec( + `node ${main} test --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + t.match(stdout, 'Organization:', 'contains human readable output'); + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + const jsonObj = JSON.parse(outputFileContents); + const okValue = jsonObj.ok as boolean; + t.ok(okValue, 'JSON output ok'); + }, + ); +}); + +test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => { + t.plan(1); + + exec( + `node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync( + 'snyk-direct-json-test-output.json', + 'utf-8', + ); + unlinkSync('./snyk-direct-json-test-output.json'); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test('`test --json-file-output can handle a relative path`', (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`; + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); +}); + +test( + '`test --json-file-output can handle an absolute path`', + { skip: iswindows }, + (t) => { + t.plan(1); + + // if 'test-output' doesn't exist, created it + if (!existsSync('test-output')) { + mkdirSync('test-output'); + } + + const tempFolder = uuidv4(); + const outputPath = join( + process.cwd(), + `test-output/${tempFolder}/snyk-direct-json-test-output.json`, + ); + + exec( + `node ${main} test --json --json-file-output=${outputPath}`, + (err, stdout) => { + if (err) { + throw err; + } + const stdoutJson = stdout; + const outputFileContents = readFileSync(outputPath, 'utf-8'); + unlinkSync(outputPath); + rmdirSync(`test-output/${tempFolder}`); + t.equals(stdoutJson, outputFileContents); + }, + ); + }, +); From ed2aab1f81b3b631dd7da5ba98d0f5d1dfdbfbc8 Mon Sep 17 00:00:00 2001 From: Jakub Mikulas Date: Fri, 4 Sep 2020 12:29:51 +0200 Subject: [PATCH 3/4] test: mount empty fixture for Alpine smoke test --- test/smoke/README.md | 23 +++++++++++++++++------ test/smoke/alpine/Dockerfile | 1 + 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/test/smoke/README.md b/test/smoke/README.md index 9c30e7d54df..bc16c47bf8f 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -5,8 +5,9 @@ Design goal is to have a single test suite, that can detect if CLI is not workin CLI is being tested by a series of tests using [Shellspec](https://shellspec.info). See them in a `test/smoke/spec` folder. Spec in this folder is used as a -1) **"Smoke test" step in CircleCI** to verify that built CLI can run -2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working. + +1. **"Smoke test" step in CircleCI** to verify that built CLI can run +2. **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working. ## How to add a new smoke test @@ -23,11 +24,21 @@ To run these tests locally: - [Shellspec](https://shellspec.info) - [jq](https://stedolan.github.io/jq/) - timeout (if not available on your platform) + 2. Run: - ```sh - cd test/smoke - CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d - ``` + +```sh +cd test/smoke +CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d +``` + +To run the Alpine test in Docker locally: + +``` + docker build -f ./test/smoke/alpine/Dockerfile -t snyk-cli-alpine ./test/ && docker run --rm -eCI=1 -eSMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN snyk-cli-alpine +``` + +_Note: Alpine image is not copying/mounting everything, so you might need to add anything new to the `test/smoke/alpine/Dockerfile`_ ## TODO diff --git a/test/smoke/alpine/Dockerfile b/test/smoke/alpine/Dockerfile index 2c83c730754..5a6734e1b49 100644 --- a/test/smoke/alpine/Dockerfile +++ b/test/smoke/alpine/Dockerfile @@ -2,6 +2,7 @@ FROM shellspec/shellspec:latest COPY ./smoke/ /snyk/smoke/ COPY ./fixtures/basic-npm/ /snyk/fixtures/basic-npm/ +COPY ./fixtures/empty/ /snyk/fixtures/empty/ RUN shellspec --version RUN apk add curl jq libgcc libstdc++ From 7c76be159c5f7f8056d273dd5b77b63c8b1b66fc Mon Sep 17 00:00:00 2001 From: ghe Date: Fri, 4 Sep 2020 12:05:49 +0100 Subject: [PATCH 4/4] chore: update helptext with new exit code --- help/help.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/help/help.txt b/help/help.txt index a0aa75dd310..a1de892ed73 100644 --- a/help/help.txt +++ b/help/help.txt @@ -175,6 +175,7 @@ Possible exit statuses and their meaning: - 0: success, no vulns found - 1: action_needed, vulns found - 2: failure, try to re-run command + - 3: failure, no supported projects detected Pro tip: use `snyk test` in your test scripts, if a vulnerability is found, the process will exit with a non-zero exit code.