Retry npm install in CI (#1320)

.circleci/config.yml

@@ -1,57 +1,50 @@
----
-# "Include" for unit tests definition.
-
-unit_tests: &unit_tests
-  steps:
-    - checkout
-    - run: npm install
-    - run: npm test
-    - run:
-        name: Submit coverage data to codecov.
-        command: node_modules/.bin/codecov
-        when: always
-
-# Testing the samples requires running `npm link` and
-# `npm link googleapis`, which attempts to make a global
-# change.  This leads to a permissions error unless the
-# global npm module directory is changed.  These samples
-# only tested against latest LTS.
-test_samples: &test_samples
-  steps:
-    - checkout
-    - run: npm install
-    - run: mkdir ~/.npm-packages
-    - run: echo "prefix = $HOME/.npm-packages" >> ~/.npmrc
-    - run: export PATH=~/.npm-packages/bin:$PATH
-    - run: make && make test-samples
-
-version: 2.0
+version: 2
 workflows:
   version: 2
   tests:
     jobs: &workflow_jobs
       - node6:
-          filters:
+          filters: &all_commits
             tags:
               only: /.*/
       - node8:
-          filters:
-            tags:
-              only: /.*/
+          filters: *all_commits
       - node10:
-          filters:
-            tags:
-              only: /.*/
-      - publish_npm:
+          filters: *all_commits
+      - lint:
+          requires:
+            - node6
+            - node8
+            - node10
+          filters: *all_commits
+      - docs:
           requires:
             - node6
             - node8
             - node10
+          filters: *all_commits
+      - system_tests:
+          requires:
+            - lint
+            - docs
+          filters: &master_and_releases
+            branches:
+              only: master
+            tags: &releases
+              only: '/^v[\d.]+$/'
+      - sample_tests:
+          requires:
+            - lint
+            - docs
+          filters: *master_and_releases
+      - publish_npm:
+          requires:
+            - system_tests
+            - sample_tests
           filters:
             branches:
               ignore: /.*/
-            tags:
-              only: /^v[\d.]+$/
+            tags: *releases
   nightly:
     triggers:
       - schedule:
@@ -60,36 +53,125 @@ workflows:
             branches:
               only: master
     jobs: *workflow_jobs
-
 jobs:
   node6:
     docker:
-      - image: node:6
+      - image: 'node:6'
         user: node
-    <<: *unit_tests
+    steps: &unit_tests_steps
+      - checkout
+      - run: &npm_install_and_link
+          name: Install and link the module
+          command: |-
+            mkdir -p /home/node/.npm-global
+            ./.circleci/npm-install-retry.js
+          environment:
+            NPM_CONFIG_PREFIX: /home/node/.npm-global
+      - run: npm test
+      - run: node_modules/.bin/codecov
+
   node8:
     docker:
-      - image: node:8
+      - image: 'node:8'
         user: node
-    <<: *unit_tests
-    <<: *test_samples
+    steps: *unit_tests_steps
   node10:
     docker:
-      - image: node:10
+      - image: 'node:10'
         user: node
-    <<: *unit_tests
-
-  publish_npm:
+    steps: *unit_tests_steps
+  lint:
     docker:
-      - image: node:8
+      - image: 'node:8'
         user: node
     steps:
       - checkout
-      - run: npm install
+      - run: *npm_install_and_link
+      - run: &samples_npm_install_and_link
+          name: Link the module being tested to the samples.
+          command: |
+            cd samples/
+            npm link ../
+            ./../.circleci/npm-install-retry.js
+          environment:
+            NPM_CONFIG_PREFIX: /home/node/.npm-global
       - run:
-          name: Build
-          command: make build
-      - run: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc
+          name: Run linting.
+          command: npm run lint
+          environment:
+            NPM_CONFIG_PREFIX: /home/node/.npm-global
+  docs:
+    docker:
+      - image: 'node:8'
+        user: node
+    steps:
+      - checkout
+      - run: *npm_install_and_link
+      - run: npm run docs
+  sample_tests:
+    docker:
+      - image: 'node:8'
+        user: node
+    steps:
+      - checkout
       - run:
-         name: Publish the module to npm.
-         command: npm publish --access=public
+          name: Decrypt credentials.
+          command: |
+            if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then
+              openssl aes-256-cbc -d -in .circleci/key.json.enc \
+                -out .circleci/key.json \
+                -k "${SYSTEM_TESTS_ENCRYPTION_KEY}"
+            fi
+      - run: *npm_install_and_link
+      - run: *samples_npm_install_and_link
+      - run:
+          name: Run sample tests.
+          command: npm run samples-test
+          environment:
+            GCLOUD_PROJECT: long-door-651
+            GOOGLE_APPLICATION_CREDENTIALS: /home/node/samples/.circleci/key.json
+            NPM_CONFIG_PREFIX: /home/node/.npm-global
+      - run:
+          name: Remove unencrypted key.
+          command: |
+            if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then
+              rm .circleci/key.json
+            fi
+          when: always
+    working_directory: /home/node/samples/
+  system_tests:
+    docker:
+      - image: 'node:8'
+        user: node
+    steps:
+      - checkout
+      - run:
+          name: Decrypt credentials.
+          command: |
+            if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then
+              openssl aes-256-cbc -d -in .circleci/key.json.enc \
+                -out .circleci/key.json \
+                -k "${SYSTEM_TESTS_ENCRYPTION_KEY}"
+            fi
+      - run: *npm_install_and_link
+      - run:
+          name: Run system tests.
+          command: npm run system-test
+          environment:
+            GOOGLE_APPLICATION_CREDENTIALS: .circleci/key.json
+      - run:
+          name: Remove unencrypted key.
+          command: |
+            if ! [[ -z "${SYSTEM_TESTS_ENCRYPTION_KEY}" ]]; then
+              rm .circleci/key.json
+            fi
+          when: always
+  publish_npm:
+    docker:
+      - image: 'node:8'
+        user: node
+    steps:
+      - checkout
+      - run: ./.circleci/npm-install-retry.js
+      - run: echo "//registry.npmjs.org/:_authToken=${NPM_TOKEN}" > ~/.npmrc
+      - run: npm publish --access=public

.circleci/npm-install-retry.js

@@ -0,0 +1,60 @@
+#!/usr/bin/env node
+
+let spawn = require('child_process').spawn;
+
+//
+//USE: ./index.js <ms npm can be idle> <number of attempts> [... NPM ARGS]
+//
+
+let timeout = process.argv[2] || 60000;
+let attempts = process.argv[3] || 3;
+let args = process.argv.slice(4);
+if (args.length === 0) {
+  args = ['install'];
+}
+
+(function npm() {
+  let timer;
+  args.push('--verbose');
+  let proc = spawn('npm', args);
+  proc.stdout.pipe(process.stdout);
+  proc.stderr.pipe(process.stderr);
+  proc.stdin.end();
+  proc.stdout.on('data', () => {
+    setTimer();
+  });
+  proc.stderr.on('data', () => {
+    setTimer();
+  });
+
+  // side effect: this also restarts when npm exits with a bad code even if it
+  // didnt timeout
+  proc.on('close', (code, signal) => {
+    clearTimeout(timer);
+    if (code || signal) {
+      console.log('[npm-are-you-sleeping] npm exited with code ' + code + '');
+
+      if (--attempts) {
+        console.log('[npm-are-you-sleeping] restarting');
+        npm();
+      } else {
+        console.log('[npm-are-you-sleeping] i tried lots of times. giving up.');
+        throw new Error("npm install fails");
+      }
+    }
+  });
+
+  function setTimer() {
+    clearTimeout(timer);
+    timer = setTimeout(() => {
+      console.log('[npm-are-you-sleeping] killing npm with SIGTERM');
+      proc.kill('SIGTERM');
+      // wait a couple seconds
+      timer = setTimeout(() => {
+        // its it's still not closed sigkill
+        console.log('[npm-are-you-sleeping] killing npm with SIGKILL');
+        proc.kill('SIGKILL');
+      }, 2000);
+    }, timeout);
+  }
+})();