chore: migrate remaining tests to Mocha (#5616)

This commit updates all the non-Puppeteer unit tests to run using Mocha and then deletes the custom test runner framework from this repository. The documentation has also been updated.

.eslintignore

@@ -2,9 +2,7 @@ test/assets/modernizr.js
 third_party/*
 utils/browser/puppeteer-web.js
 utils/doclint/check_public_api/test/
-utils/testrunner/examples/
 node6/*
 node6-test/*
-node6-testrunner/*
 experimental/
 lib/

CONTRIBUTING.md

@@ -148,8 +148,8 @@ A barrier for introducing new installation dependencies is especially high:
 - Tests should be *hermetic*. Tests should not depend on external services.
 - Tests should work on all three platforms: Mac, Linux and Win. This is especially important for screenshot tests.
 
-Puppeteer tests are located in [`test/test.js`](https://github.com/puppeteer/puppeteer/blob/master/test/test.js)
-and are written with a [TestRunner](https://github.com/puppeteer/puppeteer/tree/master/utils/testrunner) framework.
+Puppeteer tests are located in the test directory ([`test`](https://github.com/puppeteer/puppeteer/blob/master/test/) and are written using Mocha. See [`test/README.md`](https://github.com/puppeteer/puppeteer/blob/master/test/) for more details.
+
 Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected.
 
 - To run all tests:

mocha-config/base.js

@@ -0,0 +1,3 @@
+module.exports = {
+  reporter: 'dot',
+};

mocha-config/browser-bundle-tests.js

@@ -0,0 +1,6 @@
+const base = require('./base');
+
+module.exports = {
+  ...base,
+  spec: 'utils/browser/*.spec.js',
+};

mocha-config/doclint-tests.js

@@ -0,0 +1,6 @@
+const base = require('./base');
+
+module.exports = {
+  ...base,
+  spec: 'utils/doclint/**/*.spec.js',
+};

.mocharc.js → mocha-config/puppeteer-unit-tests.js

@@ -1,6 +1,8 @@
+const base = require('./base');
+
 module.exports = {
+  ...base,
   file: ['./test/mocha-utils.js'],
   spec: 'test/*.spec.js',
-  reporter: 'dot',
   timeout: process.env.PUPPETEER_PRODUCT === 'firefox' ? 15 * 1000 : 10 * 1000,
 };

package.json

@@ -12,12 +12,12 @@
     "firefox_revision": "latest"
   },
   "scripts": {
-    "unit": "mocha --config .mocharc.js",
+    "unit": "mocha --config mocha-config/puppeteer-unit-tests.js",
     "coverage": "cross-env COVERAGE=1 npm run unit",
     "funit": "PUPPETEER_PRODUCT=firefox npm run unit",
     "debug-unit": "node --inspect-brk test/test.js",
-    "test-doclint": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js",
-    "test": "npm run tsc && npm run lint --silent && npm run coverage && npm run test-doclint && npm run test-types && node utils/testrunner/test/test.js",
+    "test-doclint": "mocha --config mocha-config/doclint-tests.js",
+    "test": "npm run tsc && npm run lint --silent && npm run coverage && npm run test-doclint && npm run test-types",
     "prepublishOnly": "npm run tsc",
     "dev-install": "npm run tsc && node install.js",
     "install": "node install.js",
@@ -27,7 +27,7 @@
     "apply-next-version": "node utils/apply_next_version.js",
     "bundle": "npm run tsc && npx browserify -r ./index.js:puppeteer -o utils/browser/puppeteer-web.js",
     "test-types": "node utils/doclint/generate_types && tsc --version && tsc -p utils/doclint/generate_types/test/",
-    "unit-bundle": "node utils/browser/test.js",
+    "unit-bundle": "mocha --config mocha-config/browser-bundle-tests.js",
     "update-protocol-d-ts": "node utils/protocol-types-generator"
   },
   "author": "The Chromium Authors",

test/README.md

@@ -32,3 +32,48 @@ There is also `describeChromeOnly` which will only execute the test if running i
 
 [Mocha]: https://mochajs.org/
 [Expect]: https://www.npmjs.com/package/expect
+
+## Running tests
+
+Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected.
+
+- To run all tests:
+
+```bash
+npm run unit
+```
+
+- To run a specific test, substitute the `it` with `it.only`:
+
+```js
+  ...
+  it.only('should work', async function() {
+    const {server, page} = getTestState();
+    const response = await page.goto(server.EMPTY_PAGE);
+    expect(response.ok).toBe(true);
+  });
+```
+
+- To disable a specific test, substitute the `it` with `xit` (mnemonic rule: '*cross it*'):
+
+```js
+  ...
+  // Using "xit" to skip specific test
+  xit('should work', async function({server, page}) {
+    const {server, page} = getTestState();
+    const response = await page.goto(server.EMPTY_PAGE);
+    expect(response.ok).toBe(true);
+  });
+```
+
+- To run tests in non-headless mode:
+
+```bash
+HEADLESS=false npm run unit
+```
+
+- To run tests with custom browser executable:
+
+```bash
+BINARY=<path-to-executable> npm run unit
+```

utils/browser/test.js → utils/browser/browser.spec.js

@@ -2,20 +2,16 @@ const path = require('path');
 const fs = require('fs');
 const puppeteer = require('../..');
 const {TestServer} = require('../testserver/');
-const {TestRunner, Reporter} = require('../testrunner/');
 const expect = require('expect');
 
 const puppeteerWebPath = path.join(__dirname, 'puppeteer-web.js');
 if (!fs.existsSync(puppeteerWebPath))
   throw new Error(`puppeteer-web is not built; run "npm run bundle"`);
 const puppeteerWeb = fs.readFileSync(puppeteerWebPath, 'utf8');
 
-const testRunner = new TestRunner();
-const {describe, fdescribe, xdescribe} = testRunner;
-const {it, xit, fit} = testRunner;
-const {afterAll, beforeAll, afterEach, beforeEach} = testRunner;
+const state = {};
 
-beforeAll(async state => {
+before(async() => {
   const assetsPath = path.join(__dirname, '..', '..', 'test', 'assets');
   const port = 8998;
   state.server = await TestServer.create(assetsPath, port);
@@ -26,7 +22,7 @@ beforeAll(async state => {
   state.browser = await puppeteer.launch();
 });
 
-afterAll(async state => {
+after(async() => {
   await Promise.all([
     state.server.stop(),
     state.browser.close()
@@ -35,21 +31,22 @@ afterAll(async state => {
   state.server = null;
 });
 
-beforeEach(async state => {
+beforeEach(async() => {
   state.page = await state.browser.newPage();
   await state.page.evaluateOnNewDocument(puppeteerWeb);
   await state.page.addScriptTag({
     content: puppeteerWeb + '\n//# sourceURL=puppeteer-web.js'
   });
 });
 
-afterEach(async state => {
+afterEach(async() => {
   await state.page.close();
   state.page = null;
 });
 
 describe('Puppeteer-Web', () => {
-  it('should work over web socket', async({page, serverConfig}) => {
+  it('should work over web socket', async() => {
+    const {page, serverConfig} = state;
     const browser2 = await puppeteer.launch();
     // Use in-page puppeteer to create a new page and navigate it to the EMPTY_PAGE
     await page.evaluate(async(browserWSEndpoint, serverConfig) => {
@@ -65,7 +62,8 @@ describe('Puppeteer-Web', () => {
     ]);
     await browser2.close();
   });
-  it('should work over exposed DevTools protocol', async({browser, page, serverConfig}) => {
+  it('should work over exposed DevTools protocol', async() => {
+    const {browser, page, serverConfig} = state;
     // Expose devtools protocol binding into page.
     const session = await browser.target().createCDPSession();
     const pageInfo = (await session.send('Target.getTargets')).targetInfos.find(info => info.attached);
@@ -90,11 +88,3 @@ describe('Puppeteer-Web', () => {
     ]);
   });
 });
-
-if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) {
-  console.error('ERROR: "focused" tests/suites are prohibitted on bots. Remove any "fit"/"fdescribe" declarations.');
-  process.exit(1);
-}
-
-new Reporter(testRunner);
-testRunner.run();

utils/doclint/check_public_api/test/public-api.spec.js

@@ -0,0 +1,125 @@
+/**
+ * Copyright 2017 Google Inc. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const path = require('path');
+const puppeteer = require('../../../..');
+const checkPublicAPI = require('..');
+const Source = require('../../Source');
+const mdBuilder = require('../MDBuilder');
+const jsBuilder = require('../JSBuilder');
+const expect = require('expect')
+const GoldenUtils = require('../../../../test/golden-utils');
+
+const testUtils = require('../../../../test/utils')
+
+
+describe('DocLint Public API', function() {
+  let browser;
+  let page;
+
+  before(async function() {
+    browser = await puppeteer.launch();
+    page = await browser.newPage();
+  });
+
+  after(async function() {
+    await browser.close();
+  });
+
+  describe('checkPublicAPI', function() {
+    it('diff-classes', testLint('diff-classes'));
+    it('diff-methods', testLint('diff-methods'));
+    it('diff-properties', testLint('diff-properties'));
+    it('diff-arguments', testLint('diff-arguments'));
+    it('diff-events', testLint('diff-events'));
+    it('check-duplicates', testLint('check-duplicates'));
+    it('check-sorting', testLint('check-sorting'));
+    it('check-returns', testLint('check-returns'));
+    it('js-builder-common', testJSBuilder('js-builder-common'));
+    it('js-builder-inheritance', testJSBuilder('js-builder-inheritance'));
+    it('md-builder-common', testMDBuilder('md-builder-common'));
+  });
+
+  function testLint(testName) {
+    return async () => {
+      const dirPath = path.join(__dirname, testName);
+      testUtils.extendExpectWithToBeGolden(dirPath, dirPath)
+
+      const mdSources = await Source.readdir(dirPath, '.md');
+      const jsSources = await Source.readdir(dirPath, '.js');
+      const messages = await checkPublicAPI(page, mdSources, jsSources);
+      const errors = messages.map(message => message.text);
+      expect(errors.join('\n')).toBeGolden('result.txt');
+    }
+  }
+
+  function testMDBuilder(testName) {
+    return async () => {
+      const dirPath = path.join(__dirname, testName);
+      testUtils.extendExpectWithToBeGolden(dirPath, dirPath);
+
+      const sources = await Source.readdir(dirPath, '.md');
+      const { documentation } = await mdBuilder(page, sources);
+      expect(serialize(documentation)).toBeGolden('result.txt');
+    }
+  }
+
+  function testJSBuilder(testName) {
+    return async () => {
+      const dirPath = path.join(__dirname, testName);
+      testUtils.extendExpectWithToBeGolden(dirPath, dirPath);
+
+      const sources = await Source.readdir(dirPath, '.js');
+      const { documentation } = await jsBuilder(sources);
+      expect(serialize(documentation)).toBeGolden('result.txt');
+    }
+  }
+
+  /**
+  * @param {import('../Documentation')} doc
+  */
+  function serialize(doc) {
+    const result = {
+      classes: doc.classesArray.map(cls => ({
+        name: cls.name,
+        members: cls.membersArray.map(serializeMember)
+      }))
+    };
+    return JSON.stringify(result, null, 2);
+  }
+  /**
+  * @param {import('../Documentation').Member} member
+  */
+  function serializeMember(member) {
+    return {
+      name: member.name,
+      type: serializeType(member.type),
+      kind: member.kind,
+      args: member.argsArray.length ? member.argsArray.map(serializeMember) : undefined
+    }
+  }
+  /**
+  * @param {import('../Documentation').Type} type
+  */
+  function serializeType(type) {
+    if (!type)
+      return undefined;
+    return {
+      name: type.name,
+      properties: type.properties.length ? type.properties.map(serializeMember) : undefined
+    }
+  }
+  })