test: Honeycomb system-test reporter (#19855)

* Mocha performance reporter

Co-authored-by: Tim Griesser <tgriesser10@gmail.com>

circle.yml

@@ -2088,15 +2088,19 @@ linux-workflow: &linux-workflow
         requires:
           - build
     - system-tests-chrome:
+        context: test-runner:performance-tracking
         requires:
           - system-tests-node-modules-install
     - system-tests-electron:
+        context: test-runner:performance-tracking
         requires:
           - system-tests-node-modules-install
     - system-tests-firefox:
+        context: test-runner:performance-tracking
         requires:
           - system-tests-node-modules-install
     - system-tests-non-root:
+        context: test-runner:performance-tracking
         executor: non-root-docker-user
         requires:
           - system-tests-node-modules-install

packages/server/package.json

@@ -147,6 +147,7 @@
     "@types/chai-as-promised": "7.1.2",
     "@types/chrome": "0.0.101",
     "@types/http-proxy": "1.17.4",
+    "@types/node": "14.14.31",
     "awesome-typescript-loader": "5.2.1",
     "babel-loader": "8.1.0",
     "body-parser": "1.19.0",

system-tests/lib/performance-reporter.js

@@ -0,0 +1,81 @@
+const path = require('path')
+const chalk = require('chalk')
+const Libhoney = require('libhoney')
+
+const pkg = require('@packages/root')
+const ciProvider = require('@packages/server/lib/util/ci_provider')
+const { commitInfo } = require('@cypress/commit-info')
+
+class StatsdReporter {
+  constructor (runner) {
+    if (!process.env.HONEYCOMB_API_KEY) {
+      return
+    }
+
+    console.log(chalk.green('Reporting to honeycomb'))
+
+    let branch; let commitSha
+
+    this.honey = new Libhoney({
+      dataset: 'systemtest-performance',
+      writeKey: process.env.HONEYCOMB_API_KEY,
+    })
+
+    commitInfo().then((commitInformation) => {
+      const ciInformation = ciProvider.commitParams() || {}
+
+      this.branch = commitInformation.branch || ciInformation.branch
+      this.commitSha = commitInformation.sha || ciInformation.sha
+    })
+
+    runner.on('test', (test) => {
+      test.wallclockStart = Date.now()
+    })
+
+    runner.on('test end', (test) => {
+      // Skipped tests never get a 'start' event, but they still get 'test end' somehow.
+      if (!test.state || test.state === 'skipped') {
+        return
+      }
+
+      const title = test.titlePath().join(' / ')
+      // This regex pulls apart a string like `e2e async timeouts / failing1 [electron]`
+      // into `e2e async timeouts / failing1` and `electron`, letting us use the same
+      // test name for all browsers, with the browser as a separate field.
+      // The browser capture group is optional because some tests aren't browser specific,
+      // in which case it will be undefined and not passed as a field to honeycomb.
+      const [, testTitle, browser] = title.match(/(.+?)(?: \[([a-z]+)\])?$/)
+
+      const honeycombEvent = this.honey.newEvent()
+
+      honeycombEvent.timestamp = test.wallclockStart
+      honeycombEvent.add({
+        test: testTitle,
+        specFile: path.basename(test.file),
+        browser,
+        state: test.state,
+        err: test.err && test.err.message,
+        errStack: test.err && test.err.stack,
+        durationMs: Date.now() - test.wallclockStart,
+        mochaDurationMs: test.duration,
+        branch,
+        commitSha,
+        buildUrl: process.env.CIRCLE_BUILD_URL,
+        platform: process.platform,
+        arch: process.arch,
+        version: pkg.version,
+      })
+
+      honeycombEvent.send()
+    })
+  }
+
+  // If there is no done callback, then mocha-multi-reporter will kill the process without waiting for our honeycomb post to complete.
+  done (failures, callback) {
+    if (this.honey) {
+      this.honey.flush().then(callback)
+    }
+  }
+}
+
+module.exports = StatsdReporter

system-tests/lib/spec_helper.js

@@ -103,6 +103,7 @@ beforeEach(function () {
 
   nock.disableNetConnect()
   nock.enableNetConnect(/localhost/)
+  nock.enableNetConnect(/api.honeycomb.io/)
 
   // always clean up the cache
   // before each test

system-tests/package.json

@@ -55,6 +55,7 @@
     "human-interval": "1.0.0",
     "image-size": "0.8.3",
     "lazy-ass": "1.6.0",
+    "libhoney": "3.0.0",
     "lodash": "^4.17.21",
     "mocha": "7.1.0",
     "mocha-banner": "1.1.2",

system-tests/scripts/mocha-reporter-config.json

@@ -0,0 +1,6 @@
+{
+  "reporterEnabled": "spec, mocha-junit-reporter, ../../system-tests/lib/performance-reporter.js",
+  "mochaJunitReporterReporterOptions": {
+    "mochaFile": "/tmp/cypress/junit/test-results.[hash].xml"
+  }
+}

system-tests/scripts/run.js

@@ -109,7 +109,7 @@ commandAndArguments.args.push(
   '--reporter',
   'mocha-multi-reporters',
   '--reporter-options',
-  'configFile=../../mocha-reporter-config.json',
+  'configFile=../../system-tests/scripts/mocha-reporter-config.json',
   '--extension=js,ts',
   // restore mocha 2.x behavior to force end process after spec run
   '--exit',