Merge pull request #1319 from snyk/feat/smoke-test

chore(test): extend smoke test for auth

.github/workflows/smoke-tests.yml

@@ -82,6 +82,13 @@ jobs:
           which shellspec
           shellspec --version
 
+      - name: Install brew on macOS
+        if: ${{ matrix.os == 'macos-latest' }}
+        # We need "timeout" util and we'll use brew to check our brew package as well
+        run: |
+          /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/master/install.sh)"
+          brew install coreutils
+
       - name: Install Shellspec - Windows
         shell: powershell
         if: ${{ matrix.os == 'windows-latest' }}

test/smoke/README.md

@@ -14,6 +14,12 @@ Before you start adding specs, those files are bash scripts, it's recommended to
 
 It's recommended to have a branch named `feat/smoke-test`, as [this branch will run the GitHub Action](https://github.com/snyk/snyk/blob/f35f39e96ef7aa69b22a846315dda015b12a4564/.github/workflows/smoke-tests.yml#L3-L5).
 
+To run these tests locally, install shellspec, cd into `test/smoke` folder and run:
+
+```sh
+CI=1 SMOKE_TESTS_SNYK_TOKEN=$SNYK_API_TOKEN shellspec -f d
+```
+
 ## TODO
 
 ### Missing scenarios

test/smoke/spec/snyk_auth_spec.sh

@@ -0,0 +1,46 @@
+#shellcheck shell=sh
+
+Describe "Snyk CLI Authorization"
+  After snyk_logout
+
+  It "fails when run in CI without token set"
+    When run snyk auth
+    The output should include "Snyk is missing auth token in order to run inside CI"
+    The status should be failure
+    # TODO: unusable with our current docker issues
+    The stderr should equal ""
+  End
+
+  Describe "auth outside of CI environment"
+    Before disable_is_ci_flags
+    After restore_is_ci_flags
+
+    It "fails when run without token set"
+      # Using timeout to not wait for browser confirmation
+      When run timeout 5 snyk auth
+      The output should include "Now redirecting you to our auth page, go ahead and log in,"
+      The result of function verify_login_url should include "snyk.io/login?token=" # URL found
+      The status should be failure
+      # TODO: unusable with our current docker issues
+      The stderr should equal ""
+    End
+  End
+
+
+  It "fails if given bogus token"
+    When run snyk auth abc123
+    The output should include "Authentication failed. Please check the API token"
+    The status should be failure
+    # TODO: unusable with our current docker issues
+    The stderr should equal ""
+  End
+
+  It "updates config file if given legit token"
+    When run snyk auth "${SMOKE_TESTS_SNYK_TOKEN}"
+    The output should include "Your account has been authenticated. Snyk is now ready to be used."
+    The status should be success
+    # TODO: unusable with our current docker issues
+    The stderr should equal ""
+    The result of "print_snyk_config()" should include "api: ${SMOKE_TESTS_SNYK_TOKEN}"
+  End
+End

test/smoke/spec/snyk_basic_spec.sh

@@ -51,23 +51,4 @@ Describe "Snyk CLI basics"
       The result of "print_snyk_config()" should not include "newvalue"
     End
   End
-
-  Describe "snyk auth"
-    It "fails if given bogus token"
-      When run snyk auth abc123
-      The output should include "Authentication failed. Please check the API token"
-      The status should be failure
-      # TODO: unusable with our current docker issues
-      The stderr should equal ""
-    End
-
-    It "updates config file if given legit token"
-      When run snyk auth "${SMOKE_TESTS_SNYK_TOKEN}"
-      The output should include "Your account has been authenticated. Snyk is now ready to be used."
-      The status should be success
-      # TODO: unusable with our current docker issues
-      The stderr should equal ""
-      The result of "print_snyk_config()" should include "api: ${SMOKE_TESTS_SNYK_TOKEN}"
-    End
-  End
 End

test/smoke/spec/snyk_test_spec.sh

@@ -1,6 +1,9 @@
 #shellcheck shell=sh
 
 Describe "Snyk test command"
+  Before snyk_login
+  After snyk_logout
+
   Describe "basic npm test"
     It "finds vulns in a project"
       When run snyk test "../fixtures/basic-npm"

test/smoke/spec/spec_helper.sh

@@ -1,5 +1,5 @@
 #shellcheck shell=sh
-# set -eu
+set -e
 
 print_snyk_config() {
   snyk config
@@ -8,3 +8,24 @@ print_snyk_config() {
 snyk_login() {
   snyk auth "${SMOKE_TESTS_SNYK_TOKEN}"
 }
+
+snyk_logout() {
+  snyk config clear
+}
+
+verify_login_url() {
+  # https://snyk.io/login?token=uuid-token&utm_medium=cli&utm_source=cli&utm_campaign=cli&os=darwin&docker=false
+  echo "$1" | grep https | grep -E "^https://(dev\.)?(test\.)?snyk\.io/login\?token=[a-z0-9]{8}-([a-z0-9]{4}-){3}[a-z0-9]{12}\&.*$"
+}
+
+# These 2 commands should run in succession, some CLI functionality uses isCI detection
+disable_is_ci_flags() {
+  # save original value and unset
+  if [ -n "${CI}" ]; then CI_BACKUP_VALUE=$CI; unset CI; fi
+  if [ -n "${CIRCLECI}" ]; then CIRCLECI_BACKUP_VALUE=$CIRCLECI; unset CIRCLECI; fi
+}
+restore_is_ci_flags() {
+  # recover the original value
+  if [ -n "${CI}" ]; then CI=$CI_BACKUP_VALUE; unset CI_BACKUP_VALUE; fi
+  if [ -n "${CIRCLECI}" ]; then CIRCLECI=$CIRCLECI_BACKUP_VALUE; unset CIRCLECI_BACKUP_VALUE; fi
+}