Merge pull request #1327 from snyk/feat/smoke-test

chore(test): add regression test for valid JSON bodies

.circleci/config.yml

@@ -75,6 +75,7 @@ commands:
     steps:
       - run: curl -fsSL https://git.io/shellspec | sh -s -- -y
       - run: sudo ln -s ${HOME}/.local/lib/shellspec/shellspec /usr/local/bin/shellspec
+      - run: sudo apt-get install jq
   show_node_version:
     description: Log Node and npm version
     steps:

.github/workflows/smoke-tests.yml

@@ -84,10 +84,22 @@ jobs:
 
       - name: Install brew on macOS
         if: ${{ matrix.os == 'macos-latest' }}
-        # We need "timeout" util and we'll use brew to check our brew package as well
+        # We need "timeout" and "jq" util and we'll use brew to check our brew package as well
         run: |
           /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
           brew install coreutils
+          brew install jq
+
+      - name: Install scoop on Windows
+        if: ${{ matrix.os == 'windows-latest'}}
+        run: |
+          iwr -useb get.scoop.sh | iex
+          scoop install jq
+
+      - name: Install jq on Ubuntu
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt-get install jq
 
       - name: Install Shellspec - Windows
         shell: powershell

test/smoke/README.md

@@ -14,9 +14,16 @@ Before you start adding specs, those files are bash scripts, it's recommended to
 
 It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5).
 
-To run these tests locally, install shellspec, cd into `test/smoke` folder and run:
+To run these tests locally, install:
+
+- [Shellspec](https://shellspec.info)
+- [jq](https://stedolan.github.io/jq/)
+- timeout (if not available on your platform)
+
+cd into `test/smoke` folder and run:
 
 ```sh
+cd test/smoke
 CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
 ```
 
@@ -26,7 +33,8 @@ CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
 
 - [x] basics: version, help, config
 - [x] auth [TOKEN]
-- [ ] test [--json][npm project, java-goof]
+- [x] test [--json][npm project]
+- [ ] test [--json][java-goof]
 - [ ] policy, ignore
 - [ ] monitor
 - [ ] wizard - possibly impossible? (maybe a basic test that it even loads)

test/smoke/spec/sanity_spec.sh

@@ -0,0 +1,54 @@
+#shellcheck shell=sh
+
+: '
+  Since we are dealing with multiple utilities and environments
+  we should have a sanity test in place to test them
+'
+
+Describe "Snyk CLI"
+  It "have Snyk CLI available"
+    When run which snyk
+    The output should include "/snyk"
+    The status should be success
+    The stderr should equal ""
+  End
+End
+
+Describe "sanity checks for tooling"
+  Describe "timeout"
+    It "have timeout available"
+      When run which timeout
+      The output should include "/timeout"
+      The status should be success
+      The stderr should equal ""
+    End
+  End
+
+  Describe "jq"
+    It "have jq available"
+      When run which jq
+      The output should include "/jq"
+      The status should be success
+      The stderr should equal ""
+    End
+
+    It "validates JSON"
+      When run echo '{"k": [1,2]}'
+      The result of function check_valid_json should be success
+    End
+
+    It "validates JSON when called as When-function"
+      When run check_valid_json '{"k": [1,2]}'
+      The status should be success
+      The stdout should equal 0
+      The stderr should equal ""
+    End
+
+    # Only way to capture parse error
+    It "fails on invalid JSON"
+      When run check_valid_json '{"k": [1,2'
+      The status should be failure
+      The stderr should include "parse error"
+    End
+  End
+End

test/smoke/spec/snyk_test_spec.sh

@@ -4,22 +4,59 @@ Describe "Snyk test command"
   Before snyk_login
   After snyk_logout
 
-  Describe "basic npm test"
-    It "finds vulns in a project"
-      When run snyk test "../fixtures/basic-npm"
-      The status should be failure
+  Describe "npm test"
+    run_test_in_subfolder() {
+      cd ../fixtures/basic-npm || return
+      snyk test
+    }
+
+    It "finds vulns in a project in the same folder"
+      When run run_test_in_subfolder
+      The status should be failure # issues found
+      The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
+      The stderr should equal ""
+    End
+
+    It "finds vulns in a project when pointing to a folder"
+      When run snyk test ../fixtures/basic-npm
+      The status should be failure # issues found
+      The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
+      The stderr should equal ""
+    End
+
+    It "finds vulns in a project when pointing to a file"
+      When run snyk test --file=../fixtures/basic-npm/package.json
+      The status should be failure # issues found
       The output should include "https://snyk.io/vuln/npm:minimatch:20160620"
       The stderr should equal ""
     End
   End
 
-  Describe "basic npm test with JSON output"
-    It "finds vulns in a project"
-      When run snyk test "../fixtures/basic-npm" --json
-      The status should be failure
+  Describe "npm test with JSON output"
+    It "outputs a valid JSON with vulns"
+      When run snyk test ../fixtures/basic-npm --json
+      The status should be failure # issues found
       The output should include "npm:minimatch:20160620"
-      The output should include '"vulnerabilities": [' # TODO: check valid JSON? With jq?
+      The output should include '"vulnerabilities": ['
+      The stderr should equal ""
+      The result of function check_valid_json should be success
+    End
+  End
+
+  Describe "npm test with JSON output and all-projects flag"
+    snyk_test_json_all() {
+      cd ../fixtures || return
+      snyk test --json --all-projects
+    }
+
+    # https://github.com/snyk/snyk/pull/1324
+    # Captures an issue with extra output in stderr when json flag was set and some project failed to test
+    It "won't output to stderr when one project fails and json flag is set"
+      When run snyk_test_json_all
+      The status should be failure # issues found
+      The output should include '"error": "package' # we expect some error
       The stderr should equal ""
+      The result of function check_valid_json should be success
     End
   End
 End

test/smoke/spec/spec_helper.sh

@@ -18,6 +18,12 @@ verify_login_url() {
   echo "$1" | grep https | grep -E "^https://(dev\.)?(test\.)?snyk\.io/login\?token=[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}\&.*$"
 }
 
+# Consume stdout and checks validates whether it's a valid JSON
+check_valid_json() {
+  printf %s "$1" | jq . > /dev/null
+  echo $?
+}
+
 # These 2 commands should run in succession, some CLI functionality uses isCI detection
 disable_is_ci_flags() {
   # save original value and unset