Skip to content

Commit

Permalink
Merge pull request #1386 from snyk/feat/exit-code-for-no-projects
Browse files Browse the repository at this point in the history
Feat/exit code for no projects
  • Loading branch information
lili2311 committed Sep 4, 2020
2 parents 945722f + 7c76be1 commit 4d30f5f
Show file tree
Hide file tree
Showing 9 changed files with 169 additions and 114 deletions.
1 change: 1 addition & 0 deletions help/help.txt
Expand Up @@ -175,6 +175,7 @@ Possible exit statuses and their meaning:
- 0: success, no vulns found
- 1: action_needed, vulns found
- 2: failure, try to re-run command
- 3: failure, no supported projects detected

Pro tip: use `snyk test` in your test scripts, if a vulnerability is
found, the process will exit with a non-zero exit code.
Expand Down
8 changes: 8 additions & 0 deletions src/cli/index.ts
Expand Up @@ -43,6 +43,7 @@ const debug = Debug('snyk');
const EXIT_CODES = {
VULNS_FOUND: 1,
ERROR: 2,
NO_SUPPORTED_MANIFESTS_FOUND: 3,
};

async function runCommand(args: Args) {
Expand Down Expand Up @@ -89,6 +90,13 @@ async function handleError(args, error) {
spinner.clearAll();
let command = 'bad-command';
let exitCode = EXIT_CODES.ERROR;
const noSupportedManifestsFound = error.message?.includes(
'Could not detect supported target files in',
);

if (noSupportedManifestsFound) {
exitCode = EXIT_CODES.NO_SUPPORTED_MANIFESTS_FOUND;
}

const vulnsFound = error.code === 'VULNS';
if (vulnsFound) {
Expand Down
4 changes: 3 additions & 1 deletion src/lib/snyk-test/index.js
Expand Up @@ -51,7 +51,9 @@ function executeTest(root, options) {
return results;
});
} catch (error) {
return Promise.reject(chalk.red.bold(error));
return Promise.reject(
chalk.red.bold(error.message ? error.message : error),
);
}
}

Expand Down
109 changes: 3 additions & 106 deletions test/acceptance/cli-args.test.ts
@@ -1,8 +1,6 @@
import { test } from 'tap';
import { exec } from 'child_process';
import { sep, join } from 'path';
import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
import { v4 as uuidv4 } from 'uuid';
import { sep } from 'path';

const osName = require('os-name');

Expand Down Expand Up @@ -101,7 +99,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
}
t.match(
stdout.trim(),
'CustomError: Illegal infrastructure as code target file',
'Illegal infrastructure as code target file',
'correct error output',
);
},
Expand All @@ -118,7 +116,7 @@ test('snyk test command should fail when iac file is not supported', (t) => {
}
t.match(
stdout.trim(),
'CustomError: Not supported infrastructure as code target files in',
'Not supported infrastructure as code target files in',
'correct error output',
);
},
Expand Down Expand Up @@ -346,104 +344,3 @@ test('`test --json-file-output no value produces error message`', (t) => {

optionsToTest.forEach(validate);
});

test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
t.plan(2);

exec(
`node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
t.match(stdout, 'Organization:', 'contains human readable output');
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
const jsonObj = JSON.parse(outputFileContents);
const okValue = jsonObj.ok as boolean;
t.ok(okValue, 'JSON output ok');
},
);
});

test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
t.plan(1);

exec(
`node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
t.equals(stdoutJson, outputFileContents);
},
);
});

test('`test --json-file-output can handle a relative path`', (t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
});

test(
'`test --json-file-output can handle an absolute path`',
{ skip: iswindows },
(t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = join(
process.cwd(),
`test-output/${tempFolder}/snyk-direct-json-test-output.json`,
);

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
},
);
Empty file.
25 changes: 19 additions & 6 deletions test/smoke/README.md
Expand Up @@ -4,7 +4,10 @@ Design goal is to have a single test suite, that can detect if CLI is not workin

CLI is being tested by a series of tests using [Shellspec](https://shellspec.info). See them in a `test/smoke/spec` folder.

Spec in this folder is used as a 1) **"Smoke test" step in CircleCI** to verify that built CLI can run 2) **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working.
Spec in this folder is used as a

1. **"Smoke test" step in CircleCI** to verify that built CLI can run
2. **["Smoke Tests"](https://github.com/snyk/snyk/actions?query=workflow%3A%22Smoke+Tests%22) GitHub Action** to verify that our distribution channels are working.

## How to add a new smoke test

Expand All @@ -14,19 +17,29 @@ Before you start adding specs, those files are bash scripts, it's recommended to

It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5).

To run these tests locally, install:
To run these tests locally:

1. Install:

- [Shellspec](https://shellspec.info)
- [jq](https://stedolan.github.io/jq/)
- timeout (if not available on your platform)
- [Shellspec](https://shellspec.info)
- [jq](https://stedolan.github.io/jq/)
- timeout (if not available on your platform)

cd into `test/smoke` folder and run:
2. Run:

```sh
cd test/smoke
CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
```

To run the Alpine test in Docker locally:

```
docker build -f ./test/smoke/alpine/Dockerfile -t snyk-cli-alpine ./test/ && docker run --rm -eCI=1 -eSMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN snyk-cli-alpine
```

_Note: Alpine image is not copying/mounting everything, so you might need to add anything new to the `test/smoke/alpine/Dockerfile`_

## TODO

### Wishlist
Expand Down
1 change: 1 addition & 0 deletions test/smoke/alpine/Dockerfile
Expand Up @@ -2,6 +2,7 @@ FROM shellspec/shellspec:latest

COPY ./smoke/ /snyk/smoke/
COPY ./fixtures/basic-npm/ /snyk/fixtures/basic-npm/
COPY ./fixtures/empty/ /snyk/fixtures/empty/

RUN shellspec --version
RUN apk add curl jq libgcc libstdc++
Expand Down
21 changes: 20 additions & 1 deletion test/smoke/spec/snyk_test_spec.sh
Expand Up @@ -10,9 +10,28 @@ Describe "Snyk test command"
snyk test
}

run_test_in_empty_subfolder() {
cd ../fixtures/empty || return
snyk test
}

It "throws error when file does not exist"
When run snyk test --file=non-existent/package.json
The status should equal 2
The output should include "Could not find the specified file"
The stderr should equal ""
End

It "throws error when no suppored manifests detected"
When run run_test_in_empty_subfolder
The status should equal 3
The output should include "Could not detect supported target files in"
The stderr should equal ""
End

It "finds vulns in a project in the same folder"
When run run_test_in_subfolder
The status should be failure # issues found
The status should equal 1
The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
The stderr should equal ""
End
Expand Down
114 changes: 114 additions & 0 deletions test/system/cli-json-file-output.test.ts
@@ -0,0 +1,114 @@
import { test } from 'tap';
import { exec } from 'child_process';
import { sep, join } from 'path';
import { readFileSync, unlinkSync, rmdirSync, mkdirSync, existsSync } from 'fs';
import { v4 as uuidv4 } from 'uuid';

const osName = require('os-name');

const main = './dist/cli/index.js'.replace(/\//g, sep);
const iswindows =
osName()
.toLowerCase()
.indexOf('windows') === 0;

test('`test --json-file-output can save JSON output to file while sending human readable output to stdout`', (t) => {
t.plan(2);

exec(
`node ${main} test --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
t.match(stdout, 'Organization:', 'contains human readable output');
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
const jsonObj = JSON.parse(outputFileContents);
const okValue = jsonObj.ok as boolean;
t.ok(okValue, 'JSON output ok');
},
);
});

test('`test --json-file-output produces same JSON output as normal JSON output to stdout`', (t) => {
t.plan(1);

exec(
`node ${main} test --json --json-file-output=snyk-direct-json-test-output.json`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(
'snyk-direct-json-test-output.json',
'utf-8',
);
unlinkSync('./snyk-direct-json-test-output.json');
t.equals(stdoutJson, outputFileContents);
},
);
});

test('`test --json-file-output can handle a relative path`', (t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = `test-output/${tempFolder}/snyk-direct-json-test-output.json`;

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
});

test(
'`test --json-file-output can handle an absolute path`',
{ skip: iswindows },
(t) => {
t.plan(1);

// if 'test-output' doesn't exist, created it
if (!existsSync('test-output')) {
mkdirSync('test-output');
}

const tempFolder = uuidv4();
const outputPath = join(
process.cwd(),
`test-output/${tempFolder}/snyk-direct-json-test-output.json`,
);

exec(
`node ${main} test --json --json-file-output=${outputPath}`,
(err, stdout) => {
if (err) {
throw err;
}
const stdoutJson = stdout;
const outputFileContents = readFileSync(outputPath, 'utf-8');
unlinkSync(outputPath);
rmdirSync(`test-output/${tempFolder}`);
t.equals(stdoutJson, outputFileContents);
},
);
},
);

0 comments on commit 4d30f5f

Please sign in to comment.