diff --git a/.eslintignore b/.eslintignore index 4b5508b266ee9..6e6149f2c22fc 100644 --- a/.eslintignore +++ b/.eslintignore @@ -2,9 +2,7 @@ test/assets/modernizr.js third_party/* utils/browser/puppeteer-web.js utils/doclint/check_public_api/test/ -utils/testrunner/examples/ node6/* node6-test/* -node6-testrunner/* experimental/ lib/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1e181b187adb9..e34eee92d428a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -148,8 +148,8 @@ A barrier for introducing new installation dependencies is especially high: - Tests should be *hermetic*. Tests should not depend on external services. - Tests should work on all three platforms: Mac, Linux and Win. This is especially important for screenshot tests. -Puppeteer tests are located in [`test/test.js`](https://github.com/puppeteer/puppeteer/blob/master/test/test.js) -and are written with a [TestRunner](https://github.com/puppeteer/puppeteer/tree/master/utils/testrunner) framework. +Puppeteer tests are located in the test directory ([`test`](https://github.com/puppeteer/puppeteer/blob/master/test/) and are written using Mocha. See [`test/README.md`](https://github.com/puppeteer/puppeteer/blob/master/test/) for more details. + Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected. - To run all tests: diff --git a/mocha-config/base.js b/mocha-config/base.js new file mode 100644 index 0000000000000..af830b8634e73 --- /dev/null +++ b/mocha-config/base.js @@ -0,0 +1,3 @@ +module.exports = { + reporter: 'dot', +}; diff --git a/mocha-config/browser-bundle-tests.js b/mocha-config/browser-bundle-tests.js new file mode 100644 index 0000000000000..99e9ab3abafa9 --- /dev/null +++ b/mocha-config/browser-bundle-tests.js @@ -0,0 +1,6 @@ +const base = require('./base'); + +module.exports = { + ...base, + spec: 'utils/browser/*.spec.js', +}; diff --git a/mocha-config/doclint-tests.js b/mocha-config/doclint-tests.js new file mode 100644 index 0000000000000..279e28eb17775 --- /dev/null +++ b/mocha-config/doclint-tests.js @@ -0,0 +1,6 @@ +const base = require('./base'); + +module.exports = { + ...base, + spec: 'utils/doclint/**/*.spec.js', +}; diff --git a/.mocharc.js b/mocha-config/puppeteer-unit-tests.js similarity index 78% rename from .mocharc.js rename to mocha-config/puppeteer-unit-tests.js index 32bbf7ee51d03..964f7f7f0a2a5 100644 --- a/.mocharc.js +++ b/mocha-config/puppeteer-unit-tests.js @@ -1,6 +1,8 @@ +const base = require('./base'); + module.exports = { + ...base, file: ['./test/mocha-utils.js'], spec: 'test/*.spec.js', - reporter: 'dot', timeout: process.env.PUPPETEER_PRODUCT === 'firefox' ? 15 * 1000 : 10 * 1000, }; diff --git a/package.json b/package.json index 2035cd263540e..1aceb8bfc6902 100644 --- a/package.json +++ b/package.json @@ -12,12 +12,12 @@ "firefox_revision": "latest" }, "scripts": { - "unit": "mocha --config .mocharc.js", + "unit": "mocha --config mocha-config/puppeteer-unit-tests.js", "coverage": "cross-env COVERAGE=1 npm run unit", "funit": "PUPPETEER_PRODUCT=firefox npm run unit", "debug-unit": "node --inspect-brk test/test.js", - "test-doclint": "node utils/doclint/check_public_api/test/test.js && node utils/doclint/preprocessor/test.js", - "test": "npm run tsc && npm run lint --silent && npm run coverage && npm run test-doclint && npm run test-types && node utils/testrunner/test/test.js", + "test-doclint": "mocha --config mocha-config/doclint-tests.js", + "test": "npm run tsc && npm run lint --silent && npm run coverage && npm run test-doclint && npm run test-types", "prepublishOnly": "npm run tsc", "dev-install": "npm run tsc && node install.js", "install": "node install.js", @@ -27,7 +27,7 @@ "apply-next-version": "node utils/apply_next_version.js", "bundle": "npm run tsc && npx browserify -r ./index.js:puppeteer -o utils/browser/puppeteer-web.js", "test-types": "node utils/doclint/generate_types && tsc --version && tsc -p utils/doclint/generate_types/test/", - "unit-bundle": "node utils/browser/test.js", + "unit-bundle": "mocha --config mocha-config/browser-bundle-tests.js", "update-protocol-d-ts": "node utils/protocol-types-generator" }, "author": "The Chromium Authors", diff --git a/test/README.md b/test/README.md index f69d4fe02abf2..5771932f4be24 100644 --- a/test/README.md +++ b/test/README.md @@ -32,3 +32,48 @@ There is also `describeChromeOnly` which will only execute the test if running i [Mocha]: https://mochajs.org/ [Expect]: https://www.npmjs.com/package/expect + +## Running tests + +Despite being named 'unit', these are integration tests, making sure public API methods and events work as expected. + +- To run all tests: + +```bash +npm run unit +``` + +- To run a specific test, substitute the `it` with `it.only`: + +```js + ... + it.only('should work', async function() { + const {server, page} = getTestState(); + const response = await page.goto(server.EMPTY_PAGE); + expect(response.ok).toBe(true); + }); +``` + +- To disable a specific test, substitute the `it` with `xit` (mnemonic rule: '*cross it*'): + +```js + ... + // Using "xit" to skip specific test + xit('should work', async function({server, page}) { + const {server, page} = getTestState(); + const response = await page.goto(server.EMPTY_PAGE); + expect(response.ok).toBe(true); + }); +``` + +- To run tests in non-headless mode: + +```bash +HEADLESS=false npm run unit +``` + +- To run tests with custom browser executable: + +```bash +BINARY= npm run unit +``` diff --git a/utils/browser/test.js b/utils/browser/browser.spec.js similarity index 78% rename from utils/browser/test.js rename to utils/browser/browser.spec.js index 09d5022587b2d..276d83b434029 100644 --- a/utils/browser/test.js +++ b/utils/browser/browser.spec.js @@ -2,7 +2,6 @@ const path = require('path'); const fs = require('fs'); const puppeteer = require('../..'); const {TestServer} = require('../testserver/'); -const {TestRunner, Reporter} = require('../testrunner/'); const expect = require('expect'); const puppeteerWebPath = path.join(__dirname, 'puppeteer-web.js'); @@ -10,12 +9,9 @@ if (!fs.existsSync(puppeteerWebPath)) throw new Error(`puppeteer-web is not built; run "npm run bundle"`); const puppeteerWeb = fs.readFileSync(puppeteerWebPath, 'utf8'); -const testRunner = new TestRunner(); -const {describe, fdescribe, xdescribe} = testRunner; -const {it, xit, fit} = testRunner; -const {afterAll, beforeAll, afterEach, beforeEach} = testRunner; +const state = {}; -beforeAll(async state => { +before(async() => { const assetsPath = path.join(__dirname, '..', '..', 'test', 'assets'); const port = 8998; state.server = await TestServer.create(assetsPath, port); @@ -26,7 +22,7 @@ beforeAll(async state => { state.browser = await puppeteer.launch(); }); -afterAll(async state => { +after(async() => { await Promise.all([ state.server.stop(), state.browser.close() @@ -35,7 +31,7 @@ afterAll(async state => { state.server = null; }); -beforeEach(async state => { +beforeEach(async() => { state.page = await state.browser.newPage(); await state.page.evaluateOnNewDocument(puppeteerWeb); await state.page.addScriptTag({ @@ -43,13 +39,14 @@ beforeEach(async state => { }); }); -afterEach(async state => { +afterEach(async() => { await state.page.close(); state.page = null; }); describe('Puppeteer-Web', () => { - it('should work over web socket', async({page, serverConfig}) => { + it('should work over web socket', async() => { + const {page, serverConfig} = state; const browser2 = await puppeteer.launch(); // Use in-page puppeteer to create a new page and navigate it to the EMPTY_PAGE await page.evaluate(async(browserWSEndpoint, serverConfig) => { @@ -65,7 +62,8 @@ describe('Puppeteer-Web', () => { ]); await browser2.close(); }); - it('should work over exposed DevTools protocol', async({browser, page, serverConfig}) => { + it('should work over exposed DevTools protocol', async() => { + const {browser, page, serverConfig} = state; // Expose devtools protocol binding into page. const session = await browser.target().createCDPSession(); const pageInfo = (await session.send('Target.getTargets')).targetInfos.find(info => info.attached); @@ -90,11 +88,3 @@ describe('Puppeteer-Web', () => { ]); }); }); - -if (process.env.CI && testRunner.hasFocusedTestsOrSuites()) { - console.error('ERROR: "focused" tests/suites are prohibitted on bots. Remove any "fit"/"fdescribe" declarations.'); - process.exit(1); -} - -new Reporter(testRunner); -testRunner.run(); diff --git a/utils/doclint/check_public_api/test/public-api.spec.js b/utils/doclint/check_public_api/test/public-api.spec.js new file mode 100644 index 0000000000000..51fedbc77e4c4 --- /dev/null +++ b/utils/doclint/check_public_api/test/public-api.spec.js @@ -0,0 +1,125 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const path = require('path'); +const puppeteer = require('../../../..'); +const checkPublicAPI = require('..'); +const Source = require('../../Source'); +const mdBuilder = require('../MDBuilder'); +const jsBuilder = require('../JSBuilder'); +const expect = require('expect') +const GoldenUtils = require('../../../../test/golden-utils'); + +const testUtils = require('../../../../test/utils') + + +describe('DocLint Public API', function() { + let browser; + let page; + + before(async function() { + browser = await puppeteer.launch(); + page = await browser.newPage(); + }); + + after(async function() { + await browser.close(); + }); + + describe('checkPublicAPI', function() { + it('diff-classes', testLint('diff-classes')); + it('diff-methods', testLint('diff-methods')); + it('diff-properties', testLint('diff-properties')); + it('diff-arguments', testLint('diff-arguments')); + it('diff-events', testLint('diff-events')); + it('check-duplicates', testLint('check-duplicates')); + it('check-sorting', testLint('check-sorting')); + it('check-returns', testLint('check-returns')); + it('js-builder-common', testJSBuilder('js-builder-common')); + it('js-builder-inheritance', testJSBuilder('js-builder-inheritance')); + it('md-builder-common', testMDBuilder('md-builder-common')); + }); + + function testLint(testName) { + return async () => { + const dirPath = path.join(__dirname, testName); + testUtils.extendExpectWithToBeGolden(dirPath, dirPath) + + const mdSources = await Source.readdir(dirPath, '.md'); + const jsSources = await Source.readdir(dirPath, '.js'); + const messages = await checkPublicAPI(page, mdSources, jsSources); + const errors = messages.map(message => message.text); + expect(errors.join('\n')).toBeGolden('result.txt'); + } + } + + function testMDBuilder(testName) { + return async () => { + const dirPath = path.join(__dirname, testName); + testUtils.extendExpectWithToBeGolden(dirPath, dirPath); + + const sources = await Source.readdir(dirPath, '.md'); + const { documentation } = await mdBuilder(page, sources); + expect(serialize(documentation)).toBeGolden('result.txt'); + } + } + + function testJSBuilder(testName) { + return async () => { + const dirPath = path.join(__dirname, testName); + testUtils.extendExpectWithToBeGolden(dirPath, dirPath); + + const sources = await Source.readdir(dirPath, '.js'); + const { documentation } = await jsBuilder(sources); + expect(serialize(documentation)).toBeGolden('result.txt'); + } + } + + /** + * @param {import('../Documentation')} doc + */ + function serialize(doc) { + const result = { + classes: doc.classesArray.map(cls => ({ + name: cls.name, + members: cls.membersArray.map(serializeMember) + })) + }; + return JSON.stringify(result, null, 2); + } + /** + * @param {import('../Documentation').Member} member + */ + function serializeMember(member) { + return { + name: member.name, + type: serializeType(member.type), + kind: member.kind, + args: member.argsArray.length ? member.argsArray.map(serializeMember) : undefined + } + } + /** + * @param {import('../Documentation').Type} type + */ + function serializeType(type) { + if (!type) + return undefined; + return { + name: type.name, + properties: type.properties.length ? type.properties.map(serializeMember) : undefined + } + } + }) diff --git a/utils/doclint/check_public_api/test/test.js b/utils/doclint/check_public_api/test/test.js deleted file mode 100644 index bdf1a67f7f7ff..0000000000000 --- a/utils/doclint/check_public_api/test/test.js +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const path = require('path'); -const puppeteer = require('../../../..'); -const checkPublicAPI = require('..'); -const Source = require('../../Source'); -const mdBuilder = require('../MDBuilder'); -const jsBuilder = require('../JSBuilder'); -const GoldenUtils = require('../../../../test/golden-utils'); - -const testUtils = require('../../../../test/utils') - -const {TestRunner, Reporter} = require('../../../testrunner/'); -const expect = require('expect') -const runner = new TestRunner(); -const reporter = new Reporter(runner); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -let browser; -let page; - -beforeAll(async function() { - browser = await puppeteer.launch(); - page = await browser.newPage(); -}); - -afterAll(async function() { - await browser.close(); -}); - -describe('checkPublicAPI', function() { - it('diff-classes', testLint); - it('diff-methods', testLint); - it('diff-properties', testLint); - it('diff-arguments', testLint); - it('diff-events', testLint); - it('check-duplicates', testLint); - it('check-sorting', testLint); - it('check-returns', testLint); - it('js-builder-common', testJSBuilder); - it('js-builder-inheritance', testJSBuilder); - it('md-builder-common', testMDBuilder); -}); - -runner.run(); - -async function testLint(state, test) { - const dirPath = path.join(__dirname, test.name); - testUtils.extendExpectWithToBeGolden(dirPath, dirPath) - - const mdSources = await Source.readdir(dirPath, '.md'); - const jsSources = await Source.readdir(dirPath, '.js'); - const messages = await checkPublicAPI(page, mdSources, jsSources); - const errors = messages.map(message => message.text); - expect(errors.join('\n')).toBeGolden('result.txt'); -} - -async function testMDBuilder(state, test) { - const dirPath = path.join(__dirname, test.name); - testUtils.extendExpectWithToBeGolden(dirPath, dirPath) - - const sources = await Source.readdir(dirPath, '.md'); - const {documentation} = await mdBuilder(page, sources); - expect(serialize(documentation)).toBeGolden('result.txt'); -} - -async function testJSBuilder(state, test) { - const dirPath = path.join(__dirname, test.name); - testUtils.extendExpectWithToBeGolden(dirPath, dirPath) - - - const sources = await Source.readdir(dirPath, '.js'); - const {documentation} = await jsBuilder(sources); - expect(serialize(documentation)).toBeGolden('result.txt'); -} - -/** - * @param {import('../Documentation')} doc - */ -function serialize(doc) { - const result = { - classes: doc.classesArray.map(cls => ({ - name: cls.name, - members: cls.membersArray.map(serializeMember) - })) - }; - return JSON.stringify(result, null, 2); -} -/** - * @param {import('../Documentation').Member} member - */ -function serializeMember(member) { - return { - name: member.name, - type: serializeType(member.type), - kind: member.kind, - args: member.argsArray.length ? member.argsArray.map(serializeMember) : undefined - } -} -/** - * @param {import('../Documentation').Type} type - */ -function serializeType(type) { - if (!type) - return undefined; - return { - name: type.name, - properties: type.properties.length ? type.properties.map(serializeMember) : undefined - } -} diff --git a/utils/doclint/preprocessor/preprocessor.spec.js b/utils/doclint/preprocessor/preprocessor.spec.js new file mode 100644 index 0000000000000..951c10685e441 --- /dev/null +++ b/utils/doclint/preprocessor/preprocessor.spec.js @@ -0,0 +1,208 @@ +/** + * Copyright 2017 Google Inc. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const {runCommands, ensureReleasedAPILinks} = require('.'); +const Source = require('../Source'); +const expect = require('expect'); + +describe('doclint preprocessor specs', function() { + + describe('ensureReleasedAPILinks', function() { + it('should work with non-release version', function() { + const source = new Source('doc.md', ` + [API](https://github.com/puppeteer/puppeteer/blob/v1.1.0/docs/api.md#class-page) + `); + const messages = ensureReleasedAPILinks([source], '1.3.0-post'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` + [API](https://github.com/puppeteer/puppeteer/blob/v1.3.0/docs/api.md#class-page) + `); + }); + it('should work with release version', function() { + const source = new Source('doc.md', ` + [API](https://github.com/puppeteer/puppeteer/blob/v1.1.0/docs/api.md#class-page) + `); + const messages = ensureReleasedAPILinks([source], '1.3.0'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` + [API](https://github.com/puppeteer/puppeteer/blob/v1.3.0/docs/api.md#class-page) + `); + }); + it('should keep master links intact', function() { + const source = new Source('doc.md', ` + [API](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) + `); + const messages = ensureReleasedAPILinks([source], '1.3.0'); + expect(messages.length).toBe(0); + expect(source.text()).toBe(` + [API](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) + `); + }); + }); + + describe('runCommands', function() { + it('should throw for unknown command', function() { + const source = new Source('doc.md', ` + something + `); + const messages = runCommands([source], '1.1.1'); + expect(source.hasUpdatedText()).toBe(false); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('error'); + expect(messages[0].text).toContain('Unknown command'); + }); + describe('gen:version', function() { + it('should work', function() { + const source = new Source('doc.md', ` + Puppeteer XXX + `); + const messages = runCommands([source], '1.2.0'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` + Puppeteer v1.2.0 + `); + }); + it('should work for *-post versions', function() { + const source = new Source('doc.md', ` + Puppeteer XXX + `); + const messages = runCommands([source], '1.2.0-post'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` + Puppeteer Tip-Of-Tree + `); + }); + it('should tolerate different writing', function() { + const source = new Source('doc.md', `Puppeteer vWHAT +`); + runCommands([source], '1.1.1'); + expect(source.text()).toBe(`Puppeteer vv1.1.1`); + }); + it('should not tolerate missing gen:stop', function() { + const source = new Source('doc.md', ``); + const messages = runCommands([source], '1.2.0'); + expect(source.hasUpdatedText()).toBe(false); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('error'); + expect(messages[0].text).toContain(`Failed to find 'gen:stop'`); + }); + }); + describe('gen:empty-if-release', function() { + it('should clear text when release version', function() { + const source = new Source('doc.md', ` + XXX + `); + const messages = runCommands([source], '1.1.1'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` + + `); + }); + it('should keep text when non-release version', function() { + const source = new Source('doc.md', ` + XXX + `); + const messages = runCommands([source], '1.1.1-post'); + expect(messages.length).toBe(0); + expect(source.text()).toBe(` + XXX + `); + }); + }); + describe('gen:toc', function() { + it('should work', () => { + const source = new Source('doc.md', `XXX + ### class: page + #### page.$ + #### page.$$`); + const messages = runCommands([source], '1.3.0'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` +- [class: page](#class-page) + * [page.$](#page) + * [page.$$](#page-1) + + ### class: page + #### page.$ + #### page.$$`); + }); + it('should work with code blocks', () => { + const source = new Source('doc.md', `XXX + ### class: page + + \`\`\`bash + # yo comment + \`\`\` + `); + const messages = runCommands([source], '1.3.0'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` +- [class: page](#class-page) + + ### class: page + + \`\`\`bash + # yo comment + \`\`\` + `); + }); + it('should work with links in titles', () => { + const source = new Source('doc.md', `XXX + ### some [link](#foobar) here + `); + const messages = runCommands([source], '1.3.0'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` +- [some link here](#some-link-here) + + ### some [link](#foobar) here + `); + }); + }); + it('should work with multiple commands', function() { + const source = new Source('doc.md', ` + XXX + YYY + ZZZ + `); + const messages = runCommands([source], '1.1.1'); + expect(messages.length).toBe(1); + expect(messages[0].type).toBe('warning'); + expect(messages[0].text).toContain('doc.md'); + expect(source.text()).toBe(` + v1.1.1 + + v1.1.1 + `); + }); + }); +}); diff --git a/utils/doclint/preprocessor/test.js b/utils/doclint/preprocessor/test.js deleted file mode 100644 index d9ba46e000b80..0000000000000 --- a/utils/doclint/preprocessor/test.js +++ /dev/null @@ -1,215 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {runCommands, ensureReleasedAPILinks} = require('.'); -const Source = require('../Source'); -const expect = require('expect'); -const {TestRunner, Reporter} = require('../../testrunner/'); -const runner = new TestRunner(); -new Reporter(runner); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('ensureReleasedAPILinks', function() { - it('should work with non-release version', function() { - const source = new Source('doc.md', ` - [API](https://github.com/puppeteer/puppeteer/blob/v1.1.0/docs/api.md#class-page) - `); - const messages = ensureReleasedAPILinks([source], '1.3.0-post'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` - [API](https://github.com/puppeteer/puppeteer/blob/v1.3.0/docs/api.md#class-page) - `); - }); - it('should work with release version', function() { - const source = new Source('doc.md', ` - [API](https://github.com/puppeteer/puppeteer/blob/v1.1.0/docs/api.md#class-page) - `); - const messages = ensureReleasedAPILinks([source], '1.3.0'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` - [API](https://github.com/puppeteer/puppeteer/blob/v1.3.0/docs/api.md#class-page) - `); - }); - it('should keep master links intact', function() { - const source = new Source('doc.md', ` - [API](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) - `); - const messages = ensureReleasedAPILinks([source], '1.3.0'); - expect(messages.length).toBe(0); - expect(source.text()).toBe(` - [API](https://github.com/puppeteer/puppeteer/blob/master/docs/api.md#class-page) - `); - }); -}); - -describe('runCommands', function() { - it('should throw for unknown command', function() { - const source = new Source('doc.md', ` - something - `); - const messages = runCommands([source], '1.1.1'); - expect(source.hasUpdatedText()).toBe(false); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('error'); - expect(messages[0].text).toContain('Unknown command'); - }); - describe('gen:version', function() { - it('should work', function() { - const source = new Source('doc.md', ` - Puppeteer XXX - `); - const messages = runCommands([source], '1.2.0'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` - Puppeteer v1.2.0 - `); - }); - it('should work for *-post versions', function() { - const source = new Source('doc.md', ` - Puppeteer XXX - `); - const messages = runCommands([source], '1.2.0-post'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` - Puppeteer Tip-Of-Tree - `); - }); - it('should tolerate different writing', function() { - const source = new Source('doc.md', `Puppeteer vWHAT -`); - runCommands([source], '1.1.1'); - expect(source.text()).toBe(`Puppeteer vv1.1.1`); - }); - it('should not tolerate missing gen:stop', function() { - const source = new Source('doc.md', ``); - const messages = runCommands([source], '1.2.0'); - expect(source.hasUpdatedText()).toBe(false); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('error'); - expect(messages[0].text).toContain(`Failed to find 'gen:stop'`); - }); - }); - describe('gen:empty-if-release', function() { - it('should clear text when release version', function() { - const source = new Source('doc.md', ` - XXX - `); - const messages = runCommands([source], '1.1.1'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` - - `); - }); - it('should keep text when non-release version', function() { - const source = new Source('doc.md', ` - XXX - `); - const messages = runCommands([source], '1.1.1-post'); - expect(messages.length).toBe(0); - expect(source.text()).toBe(` - XXX - `); - }); - }); - describe('gen:toc', function() { - it('should work', () => { - const source = new Source('doc.md', `XXX - ### class: page - #### page.$ - #### page.$$`); - const messages = runCommands([source], '1.3.0'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` -- [class: page](#class-page) - * [page.$](#page) - * [page.$$](#page-1) - - ### class: page - #### page.$ - #### page.$$`); - }); - it('should work with code blocks', () => { - const source = new Source('doc.md', `XXX - ### class: page - - \`\`\`bash - # yo comment - \`\`\` - `); - const messages = runCommands([source], '1.3.0'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` -- [class: page](#class-page) - - ### class: page - - \`\`\`bash - # yo comment - \`\`\` - `); - }); - it('should work with links in titles', () => { - const source = new Source('doc.md', `XXX - ### some [link](#foobar) here - `); - const messages = runCommands([source], '1.3.0'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` -- [some link here](#some-link-here) - - ### some [link](#foobar) here - `); - }); - }); - it('should work with multiple commands', function() { - const source = new Source('doc.md', ` - XXX - YYY - ZZZ - `); - const messages = runCommands([source], '1.1.1'); - expect(messages.length).toBe(1); - expect(messages[0].type).toBe('warning'); - expect(messages[0].text).toContain('doc.md'); - expect(source.text()).toBe(` - v1.1.1 - - v1.1.1 - `); - }); -}); - -runner.run(); - diff --git a/utils/testrunner/.npmignore b/utils/testrunner/.npmignore deleted file mode 100644 index 7b4e15d4a0008..0000000000000 --- a/utils/testrunner/.npmignore +++ /dev/null @@ -1,13 +0,0 @@ -# exclude all examples and README.md -examples/ -README.md - -# repeats from .gitignore -node_modules -.npmignore -.DS_Store -*.swp -*.pyc -.vscode -package-lock.json -yarn.lock diff --git a/utils/testrunner/LICENSE b/utils/testrunner/LICENSE deleted file mode 100644 index afdfe50e72e0e..0000000000000 --- a/utils/testrunner/LICENSE +++ /dev/null @@ -1,202 +0,0 @@ - - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "[]" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright 2017 Google Inc. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/utils/testrunner/Multimap.js b/utils/testrunner/Multimap.js deleted file mode 100644 index 6e9c0769a6f2c..0000000000000 --- a/utils/testrunner/Multimap.js +++ /dev/null @@ -1,95 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -class Multimap { - constructor() { - this._map = new Map(); - } - - set(key, value) { - let set = this._map.get(key); - if (!set) { - set = new Set(); - this._map.set(key, set); - } - set.add(value); - } - - get(key) { - let result = this._map.get(key); - if (!result) - result = new Set(); - return result; - } - - has(key) { - return this._map.has(key); - } - - hasValue(key, value) { - const set = this._map.get(key); - if (!set) - return false; - return set.has(value); - } - - /** - * @return {number} - */ - get size() { - return this._map.size; - } - - delete(key, value) { - const values = this.get(key); - const result = values.delete(value); - if (!values.size) - this._map.delete(key); - return result; - } - - deleteAll(key) { - this._map.delete(key); - } - - firstValue(key) { - const set = this._map.get(key); - if (!set) - return null; - return set.values().next().value; - } - - firstKey() { - return this._map.keys().next().value; - } - - valuesArray() { - const result = []; - for (const key of this._map.keys()) - result.push(...Array.from(this._map.get(key).values())); - return result; - } - - keysArray() { - return Array.from(this._map.keys()); - } - - clear() { - this._map.clear(); - } -} - -module.exports = Multimap; diff --git a/utils/testrunner/README.md b/utils/testrunner/README.md deleted file mode 100644 index 5ae83cf15d16f..0000000000000 --- a/utils/testrunner/README.md +++ /dev/null @@ -1,61 +0,0 @@ -# TestRunner - -This test runner is used internally by Puppeteer to test Puppeteer itself. - -- testrunner is a *library*: tests are `node.js` scripts -- parallel wrt IO operations -- supports async/await -- modular -- well-isolated state per execution thread -- uses the `expect` library from `npm` for assertions. - -### Installation - -```sh -npm install --save-dev @pptr/testrunner -``` - -### Example - -Save the following as `test.js` and run using `node`: - -```sh -node test.js -``` - -```js -const {TestRunner, Reporter} = require('@pptr/testrunner'); -const expect = require('expect') - -// Runner holds and runs all the tests -const runner = new TestRunner({ - parallel: 2, // run 2 parallel threads - timeout: 1000, // setup timeout of 1 second per test -}); - -// Extract jasmine-like DSL into the global namespace -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -// Test hooks can be async. -beforeAll(async state => { - state.parallelIndex; // either 0 or 1 in this example, depending on the executing thread - state.foo = 'bar'; // set state for every test -}); - -describe('math', () => { - it('to be sane', async (state, test) => { - state.parallelIndex; // Very first test will always be ran by the 0's thread - state.foo; // this will be 'bar' - expect(2 + 2).toBe(4); - }); -}); - -// Reporter subscribes to TestRunner events and displays information in terminal -const reporter = new Reporter(runner); - -// Run all tests. -runner.run(); -``` - diff --git a/utils/testrunner/Reporter.js b/utils/testrunner/Reporter.js deleted file mode 100644 index 5e079ff4fea2f..0000000000000 --- a/utils/testrunner/Reporter.js +++ /dev/null @@ -1,245 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const RED_COLOR = '\x1b[31m'; -const GREEN_COLOR = '\x1b[32m'; -const YELLOW_COLOR = '\x1b[33m'; -const MAGENTA_COLOR = '\x1b[35m'; -const RESET_COLOR = '\x1b[0m'; - -class Reporter { - constructor(runner, options = {}) { - const { - projectFolder = null, - showSlowTests = 3, - verbose = false, - summary = true, - } = options; - this._runner = runner; - this._projectFolder = projectFolder; - this._showSlowTests = showSlowTests; - this._verbose = verbose; - this._summary = summary; - this._testCounter = 0; - runner.on('started', this._onStarted.bind(this)); - runner.on('finished', this._onFinished.bind(this)); - runner.on('teststarted', this._onTestStarted.bind(this)); - runner.on('testfinished', this._onTestFinished.bind(this)); - this._workersState = new Map(); - } - - _onStarted(runnableTests) { - this._testCounter = 0; - this._timestamp = Date.now(); - const allTests = this._runner.tests(); - if (allTests.length === runnableTests.length) - console.log(`Running all ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} tests on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`); - else - console.log(`Running ${YELLOW_COLOR}${runnableTests.length}${RESET_COLOR} focused tests out of total ${YELLOW_COLOR}${allTests.length}${RESET_COLOR} on ${YELLOW_COLOR}${this._runner.parallel()}${RESET_COLOR} worker(s):\n`); - } - - _printTermination(result, message, error) { - console.log(`${RED_COLOR}## ${result.toUpperCase()} ##${RESET_COLOR}`); - console.log('Message:'); - console.log(` ${RED_COLOR}${message}${RESET_COLOR}`); - if (error && error.stack) { - console.log('Stack:'); - console.log(error.stack.split('\n').map(line => ' ' + line).join('\n')); - } - console.log('WORKERS STATE'); - const workerIds = Array.from(this._workersState.keys()); - workerIds.sort((a, b) => a - b); - for (const workerId of workerIds) { - const {isRunning, test} = this._workersState.get(workerId); - let description = ''; - if (isRunning) - description = `${YELLOW_COLOR}RUNNING${RESET_COLOR}`; - else if (test.result === 'ok') - description = `${GREEN_COLOR}OK${RESET_COLOR}`; - else if (test.result === 'skipped') - description = `${YELLOW_COLOR}SKIPPED${RESET_COLOR}`; - else if (test.result === 'failed') - description = `${RED_COLOR}FAILED${RESET_COLOR}`; - else if (test.result === 'crashed') - description = `${RED_COLOR}CRASHED${RESET_COLOR}`; - else if (test.result === 'timedout') - description = `${RED_COLOR}TIMEDOUT${RESET_COLOR}`; - else if (test.result === 'terminated') - description = `${MAGENTA_COLOR}TERMINATED${RESET_COLOR}`; - else - description = `${RED_COLOR}${RESET_COLOR}`; - console.log(` ${workerId}: [${description}] ${test.fullName} (${formatTestLocation(test)})`); - } - process.exitCode = 2; - } - - _onFinished({result, terminationError, terminationMessage}) { - this._printTestResults(); - if (terminationMessage || terminationError) - this._printTermination(result, terminationMessage, terminationError); - process.exitCode = result === 'ok' ? 0 : 1; - } - - _printTestResults() { - // 2 newlines after completing all tests. - console.log('\n'); - - const failedTests = this._runner.failedTests(); - if (this._summary && failedTests.length > 0) { - console.log('\nFailures:'); - for (let i = 0; i < failedTests.length; ++i) { - const test = failedTests[i]; - console.log(`${i + 1}) ${test.fullName} (${formatTestLocation(test)})`); - if (test.result === 'timedout') { - console.log(' Message:'); - console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`); - } else if (test.result === 'crashed') { - console.log(' Message:'); - console.log(` ${RED_COLOR}CRASHED${RESET_COLOR}`); - } else { - console.log(' Message:'); - console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`); - console.log(' Stack:'); - if (test.error.stack) - console.log(this._beautifyStack(test.error.stack)); - } - if (test.output) { - console.log(' Output:'); - console.log(test.output.split('\n').map(line => ' ' + line).join('\n')); - } - console.log(''); - } - } - - const skippedTests = this._runner.skippedTests(); - if (this._summary && skippedTests.length > 0) { - console.log('\nSkipped:'); - for (let i = 0; i < skippedTests.length; ++i) { - const test = skippedTests[i]; - console.log(`${i + 1}) ${test.fullName} (${formatTestLocation(test)})`); - console.log(` ${YELLOW_COLOR}Temporary disabled with xit${RESET_COLOR}`); - } - } - - if (this._showSlowTests) { - const slowTests = this._runner.passedTests().sort((a, b) => { - const aDuration = a.endTimestamp - a.startTimestamp; - const bDuration = b.endTimestamp - b.startTimestamp; - return bDuration - aDuration; - }).slice(0, this._showSlowTests); - console.log(`\nSlowest tests:`); - for (let i = 0; i < slowTests.length; ++i) { - const test = slowTests[i]; - const duration = test.endTimestamp - test.startTimestamp; - console.log(` (${i + 1}) ${YELLOW_COLOR}${duration / 1000}s${RESET_COLOR} - ${test.fullName} (${formatTestLocation(test)})`); - } - } - - const tests = this._runner.tests(); - const executedTests = tests.filter(test => test.result); - const okTestsLength = executedTests.length - failedTests.length - skippedTests.length; - let summaryText = ''; - if (failedTests.length || skippedTests.length) { - const summary = [`ok - ${GREEN_COLOR}${okTestsLength}${RESET_COLOR}`]; - if (failedTests.length) - summary.push(`failed - ${RED_COLOR}${failedTests.length}${RESET_COLOR}`); - if (skippedTests.length) - summary.push(`skipped - ${YELLOW_COLOR}${skippedTests.length}${RESET_COLOR}`); - summaryText = `(${summary.join(', ')})`; - } - - console.log(`\nRan ${executedTests.length} ${summaryText} of ${tests.length} test(s)`); - const milliseconds = Date.now() - this._timestamp; - const seconds = milliseconds / 1000; - console.log(`Finished in ${YELLOW_COLOR}${seconds}${RESET_COLOR} seconds`); - } - - _beautifyStack(stack) { - if (!this._projectFolder) - return stack; - const lines = stack.split('\n').map(line => ' ' + line); - // Find last stack line that include testrunner code. - let index = 0; - while (index < lines.length && !lines[index].includes(__dirname)) - ++index; - while (index < lines.length && lines[index].includes(__dirname)) - ++index; - if (index >= lines.length) - return stack; - const line = lines[index]; - const fromIndex = line.lastIndexOf(this._projectFolder) + this._projectFolder.length; - const toIndex = line.lastIndexOf(')'); - lines[index] = line.substring(0, fromIndex) + YELLOW_COLOR + line.substring(fromIndex, toIndex) + RESET_COLOR + line.substring(toIndex); - return lines.join('\n'); - } - - _onTestStarted(test, workerId) { - this._workersState.set(workerId, {test, isRunning: true}); - } - - _onTestFinished(test, workerId) { - this._workersState.set(workerId, {test, isRunning: false}); - if (this._verbose) { - ++this._testCounter; - if (test.result === 'ok') { - console.log(`${this._testCounter}) ${GREEN_COLOR}[ OK ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'terminated') { - console.log(`${this._testCounter}) ${MAGENTA_COLOR}[ TERMINATED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'crashed') { - console.log(`${this._testCounter}) ${RED_COLOR}[ CRASHED ]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'skipped') { - console.log(`${this._testCounter}) ${YELLOW_COLOR}[SKIP]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - } else if (test.result === 'failed') { - console.log(`${this._testCounter}) ${RED_COLOR}[FAIL]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - console.log(' Message:'); - console.log(` ${RED_COLOR}${test.error.message || test.error}${RESET_COLOR}`); - console.log(' Stack:'); - if (test.error.stack) - console.log(this._beautifyStack(test.error.stack)); - if (test.output) { - console.log(' Output:'); - console.log(test.output.split('\n').map(line => ' ' + line).join('\n')); - } - } else if (test.result === 'timedout') { - console.log(`${this._testCounter}) ${RED_COLOR}[TIME]${RESET_COLOR} ${test.fullName} (${formatTestLocation(test)})`); - console.log(' Message:'); - console.log(` ${RED_COLOR}Timeout Exceeded ${this._runner.timeout()}ms${RESET_COLOR}`); - } - } else { - if (test.result === 'ok') - process.stdout.write(`${GREEN_COLOR}.${RESET_COLOR}`); - else if (test.result === 'skipped') - process.stdout.write(`${YELLOW_COLOR}*${RESET_COLOR}`); - else if (test.result === 'failed') - process.stdout.write(`${RED_COLOR}F${RESET_COLOR}`); - else if (test.result === 'crashed') - process.stdout.write(`${RED_COLOR}C${RESET_COLOR}`); - else if (test.result === 'terminated') - process.stdout.write(`${MAGENTA_COLOR}.${RESET_COLOR}`); - else if (test.result === 'timedout') - process.stdout.write(`${RED_COLOR}T${RESET_COLOR}`); - } - } -} - -function formatTestLocation(test) { - const location = test.location; - if (!location) - return ''; - return `${YELLOW_COLOR}${location.fileName}:${location.lineNumber}:${location.columnNumber}${RESET_COLOR}`; -} - -module.exports = Reporter; diff --git a/utils/testrunner/TestRunner.js b/utils/testrunner/TestRunner.js deleted file mode 100644 index 7a43d56cf0e8c..0000000000000 --- a/utils/testrunner/TestRunner.js +++ /dev/null @@ -1,472 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const path = require('path'); -const EventEmitter = require('events'); -const Multimap = require('./Multimap'); - -const TimeoutError = new Error('Timeout'); -const TerminatedError = new Error('Terminated'); - -const MAJOR_NODEJS_VERSION = parseInt(process.version.substring(1).split('.')[0], 10); - -class UserCallback { - constructor(callback, timeout) { - this._callback = callback; - this._terminatePromise = new Promise(resolve => { - this._terminateCallback = resolve; - }); - - this.timeout = timeout; - this.location = this._getLocation(); - } - - async run(...args) { - let timeoutId; - const timeoutPromise = new Promise(resolve => { - timeoutId = setTimeout(resolve.bind(null, TimeoutError), this.timeout); - }); - try { - return await Promise.race([ - Promise.resolve().then(this._callback.bind(null, ...args)).then(() => null).catch(e => e), - timeoutPromise, - this._terminatePromise - ]); - } catch (e) { - return e; - } finally { - clearTimeout(timeoutId); - } - } - - _getLocation() { - const error = new Error(); - const stackFrames = error.stack.split('\n').slice(1); - // Find first stackframe that doesn't point to this file. - for (let frame of stackFrames) { - frame = frame.trim(); - if (!frame.startsWith('at ')) - return null; - if (frame.endsWith(')')) { - const from = frame.indexOf('('); - frame = frame.substring(from + 1, frame.length - 1); - } else { - frame = frame.substring('at '.length); - } - - const match = frame.match(/^(.*):(\d+):(\d+)$/); - if (!match) - return null; - const filePath = match[1]; - const lineNumber = match[2]; - const columnNumber = match[3]; - if (filePath === __filename) - continue; - const fileName = filePath.split(path.sep).pop(); - return { fileName, filePath, lineNumber, columnNumber }; - } - return null; - } - - terminate() { - this._terminateCallback(TerminatedError); - } -} - -const TestMode = { - Run: 'run', - Skip: 'skip', - Focus: 'focus' -}; - -const TestResult = { - Ok: 'ok', - Skipped: 'skipped', // User skipped the test - Failed: 'failed', // Exception happened during running - TimedOut: 'timedout', // Timeout Exceeded while running - Terminated: 'terminated', // Execution terminated - Crashed: 'crashed', // If testrunner crashed due to this test -}; - -class Test { - constructor(suite, name, callback, declaredMode, timeout, comment) { - this.suite = suite; - this.name = name; - this.fullName = (suite.fullName + ' ' + name).trim(); - this.declaredMode = declaredMode; - this._userCallback = new UserCallback(callback, timeout); - this.location = this._userCallback.location; - this.comment = comment; - - // Test results - this.result = null; - this.error = null; - this.startTimestamp = 0; - this.endTimestamp = 0; - } -} - -class Suite { - constructor(parentSuite, name, declaredMode, comment) { - this.parentSuite = parentSuite; - this.name = name; - this.fullName = (parentSuite ? parentSuite.fullName + ' ' + name : name).trim(); - this.declaredMode = declaredMode; - this.comment = comment; - /** @type {!Array<(!Test|!Suite)>} */ - this.children = []; - - this.beforeAll = null; - this.beforeEach = null; - this.afterAll = null; - this.afterEach = null; - } -} - -class TestPass { - constructor(runner, rootSuite, tests, parallel, breakOnFailure) { - this._runner = runner; - this._parallel = parallel; - this._runningUserCallbacks = new Multimap(); - this._breakOnFailure = breakOnFailure; - - this._rootSuite = rootSuite; - this._workerDistribution = new Multimap(); - - let workerId = 0; - for (const test of tests) { - // Reset results for tests that will be run. - test.result = null; - test.error = null; - this._workerDistribution.set(test, workerId); - for (let suite = test.suite; suite; suite = suite.parentSuite) - this._workerDistribution.set(suite, workerId); - // Do not shard skipped tests across workers. - if (test.declaredMode !== TestMode.Skip) - workerId = (workerId + 1) % parallel; - } - - this._termination = null; - } - - async run() { - const terminations = [ - createTermination.call(this, 'SIGINT', TestResult.Terminated, 'SIGINT received'), - createTermination.call(this, 'SIGHUP', TestResult.Terminated, 'SIGHUP received'), - createTermination.call(this, 'SIGTERM', TestResult.Terminated, 'SIGTERM received'), - createTermination.call(this, 'unhandledRejection', TestResult.Crashed, 'UNHANDLED PROMISE REJECTION'), - ]; - for (const termination of terminations) - process.on(termination.event, termination.handler); - - const workerPromises = []; - for (let i = 0; i < this._parallel; ++i) - workerPromises.push(this._runSuite(i, [this._rootSuite], {parallelIndex: i})); - await Promise.all(workerPromises); - - for (const termination of terminations) - process.removeListener(termination.event, termination.handler); - return this._termination; - - function createTermination(event, result, message) { - return { - event, - message, - handler: error => this._terminate(result, message, error) - }; - } - } - - async _runSuite(workerId, suitesStack, state) { - if (this._termination) - return; - const currentSuite = suitesStack[suitesStack.length - 1]; - if (!this._workerDistribution.hasValue(currentSuite, workerId)) - return; - await this._runHook(workerId, currentSuite, 'beforeAll', state); - for (const child of currentSuite.children) { - if (this._termination) - break; - if (!this._workerDistribution.hasValue(child, workerId)) - continue; - if (child instanceof Test) { - await this._runTest(workerId, suitesStack, child, state); - } else { - suitesStack.push(child); - await this._runSuite(workerId, suitesStack, state); - suitesStack.pop(); - } - } - await this._runHook(workerId, currentSuite, 'afterAll', state); - } - - async _runTest(workerId, suitesStack, test, state) { - if (this._termination) - return; - this._runner._willStartTest(test, workerId); - if (test.declaredMode === TestMode.Skip) { - test.result = TestResult.Skipped; - this._runner._didFinishTest(test, workerId); - return; - } - let crashed = false; - for (let i = 0; i < suitesStack.length; i++) - crashed = (await this._runHook(workerId, suitesStack[i], 'beforeEach', state, test)) || crashed; - // If some of the beofreEach hooks error'ed - terminate this test. - if (crashed) { - test.result = TestResult.Crashed; - } else if (this._termination) { - test.result = TestResult.Terminated; - } else { - // Otherwise, run the test itself if there is no scheduled termination. - this._runningUserCallbacks.set(workerId, test._userCallback); - test.error = await test._userCallback.run(state, test); - this._runningUserCallbacks.delete(workerId, test._userCallback); - if (!test.error) - test.result = TestResult.Ok; - else if (test.error === TimeoutError) - test.result = TestResult.TimedOut; - else if (test.error === TerminatedError) - test.result = TestResult.Terminated; - else - test.result = TestResult.Failed; - } - for (let i = suitesStack.length - 1; i >= 0; i--) - crashed = (await this._runHook(workerId, suitesStack[i], 'afterEach', state, test)) || crashed; - // If some of the afterEach hooks error'ed - then this test is considered to be crashed as well. - if (crashed) - test.result = TestResult.Crashed; - this._runner._didFinishTest(test, workerId); - if (this._breakOnFailure && test.result !== TestResult.Ok) - this._terminate(TestResult.Terminated, `Terminating because a test has failed and |testRunner.breakOnFailure| is enabled`, null); - } - - async _runHook(workerId, suite, hookName, ...args) { - const hook = suite[hookName]; - if (!hook) - return false; - this._runningUserCallbacks.set(workerId, hook); - const error = await hook.run(...args); - this._runningUserCallbacks.delete(workerId, hook); - if (error === TimeoutError) { - const location = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`; - const message = `${location} - Timeout Exceeded ${hook.timeout}ms while running "${hookName}" in suite "${suite.fullName}"`; - return this._terminate(TestResult.Crashed, message, null); - } - if (error) { - const location = `${hook.location.fileName}:${hook.location.lineNumber}:${hook.location.columnNumber}`; - const message = `${location} - FAILED while running "${hookName}" in suite "${suite.fullName}"`; - return this._terminate(TestResult.Crashed, message, error); - } - return false; - } - - _terminate(result, message, error) { - if (this._termination) - return false; - this._termination = {result, message, error}; - for (const userCallback of this._runningUserCallbacks.valuesArray()) - userCallback.terminate(); - return true; - } -} - -class TestRunner extends EventEmitter { - constructor(options = {}) { - super(); - const { - timeout = 10 * 1000, // Default timeout is 10 seconds. - parallel = 1, - breakOnFailure = false, - disableTimeoutWhenInspectorIsEnabled = true, - } = options; - this._rootSuite = new Suite(null, '', TestMode.Run); - this._currentSuite = this._rootSuite; - this._tests = []; - this._timeout = timeout === 0 ? 2147483647 : timeout; - this._parallel = parallel; - this._breakOnFailure = breakOnFailure; - - this._hasFocusedTestsOrSuites = false; - - if (MAJOR_NODEJS_VERSION >= 8 && disableTimeoutWhenInspectorIsEnabled) { - const inspector = require('inspector'); - if (inspector.url()) { - console.log('TestRunner detected inspector; overriding certain properties to be debugger-friendly'); - console.log(' - timeout = 0 (Infinite)'); - this._timeout = 2147483647; - } - } - - // bind methods so that they can be used as a DSL. - this.describe = this._addSuite.bind(this, TestMode.Run, ''); - this.fdescribe = this._addSuite.bind(this, TestMode.Focus, ''); - this.xdescribe = this._addSuite.bind(this, TestMode.Skip, ''); - this.it = this._addTest.bind(this, TestMode.Run, ''); - this.fit = this._addTest.bind(this, TestMode.Focus, ''); - this.xit = this._addTest.bind(this, TestMode.Skip, ''); - this.beforeAll = this._addHook.bind(this, 'beforeAll'); - this.beforeEach = this._addHook.bind(this, 'beforeEach'); - this.afterAll = this._addHook.bind(this, 'afterAll'); - this.afterEach = this._addHook.bind(this, 'afterEach'); - } - - addTestDSL(dslName, mode, comment) { - this[dslName] = this._addTest.bind(this, mode, comment); - } - - addSuiteDSL(dslName, mode, comment) { - this[dslName] = this._addSuite.bind(this, mode, comment); - } - - _addTest(mode, comment, name, callback) { - let suite = this._currentSuite; - let isSkipped = suite.declaredMode === TestMode.Skip; - while ((suite = suite.parentSuite)) - isSkipped |= suite.declaredMode === TestMode.Skip; - const test = new Test(this._currentSuite, name, callback, isSkipped ? TestMode.Skip : mode, this._timeout, comment); - this._currentSuite.children.push(test); - this._tests.push(test); - this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus; - } - - async _addSuite(mode, comment, name, callback) { - const oldSuite = this._currentSuite; - const suite = new Suite(this._currentSuite, name, mode, comment); - this._currentSuite.children.push(suite); - this._currentSuite = suite; - const result = callback(); - if (result && (typeof result.then === 'function')) - await result; - this._currentSuite = oldSuite; - this._hasFocusedTestsOrSuites = this._hasFocusedTestsOrSuites || mode === TestMode.Focus; - } - - _addHook(hookName, callback) { - assert(this._currentSuite[hookName] === null, `Only one ${hookName} hook available per suite`); - const hook = new UserCallback(callback, this._timeout); - this._currentSuite[hookName] = hook; - } - - async run() { - const runnableTests = this._runnableTests(); - this.emit(TestRunner.Events.Started, runnableTests); - this._runningPass = new TestPass(this, this._rootSuite, runnableTests, this._parallel, this._breakOnFailure); - const termination = await this._runningPass.run(); - this._runningPass = null; - const result = {}; - if (termination) { - result.result = termination.result; - result.terminationMessage = termination.message; - result.terminationError = termination.error; - } else { - result.result = this.failedTests().length ? TestResult.Failed : TestResult.Ok; - } - this.emit(TestRunner.Events.Finished, result); - return result; - } - - terminate() { - if (!this._runningPass) - return; - this._runningPass._terminate(TestResult.Terminated, 'Terminated with |TestRunner.terminate()| call', null); - } - - timeout() { - return this._timeout; - } - - _runnableTests() { - if (!this._hasFocusedTestsOrSuites) - return this._tests; - - const tests = []; - const blacklistSuites = new Set(); - // First pass: pick "fit" and blacklist parent suites - for (const test of this._tests) { - if (test.declaredMode !== TestMode.Focus) - continue; - tests.push(test); - for (let suite = test.suite; suite; suite = suite.parentSuite) - blacklistSuites.add(suite); - } - // Second pass: pick all tests that belong to non-blacklisted "fdescribe" - for (const test of this._tests) { - let insideFocusedSuite = false; - for (let suite = test.suite; suite; suite = suite.parentSuite) { - if (!blacklistSuites.has(suite) && suite.declaredMode === TestMode.Focus) { - insideFocusedSuite = true; - break; - } - } - if (insideFocusedSuite) - tests.push(test); - } - return tests; - } - - hasFocusedTestsOrSuites() { - return this._hasFocusedTestsOrSuites; - } - - tests() { - return this._tests.slice(); - } - - failedTests() { - return this._tests.filter(test => test.result === 'failed' || test.result === 'timedout' || test.result === 'crashed'); - } - - passedTests() { - return this._tests.filter(test => test.result === 'ok'); - } - - skippedTests() { - return this._tests.filter(test => test.result === 'skipped'); - } - - parallel() { - return this._parallel; - } - - _willStartTest(test, workerId) { - test.startTimestamp = Date.now(); - this.emit(TestRunner.Events.TestStarted, test, workerId); - } - - _didFinishTest(test, workerId) { - test.endTimestamp = Date.now(); - this.emit(TestRunner.Events.TestFinished, test, workerId); - } -} - -/** - * @param {*} value - * @param {string=} message - */ -function assert(value, message) { - if (!value) - throw new Error(message); -} - -TestRunner.Events = { - Started: 'started', - Finished: 'finished', - TestStarted: 'teststarted', - TestFinished: 'testfinished', -}; - -module.exports = TestRunner; diff --git a/utils/testrunner/examples/fail.js b/utils/testrunner/examples/fail.js deleted file mode 100644 index 588f0a4feb451..0000000000000 --- a/utils/testrunner/examples/fail.js +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter} = require('..'); - -const runner = new TestRunner(); -const reporter = new Reporter(runner); -const expect = require('expect'); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - it('toBe', async (state) => { - expect(2 + 2).toBe(5); - }); - it('toBeFalsy', async (state) => { - expect(true).toBeFalsy(); - }); - it('toBeTruthy', async (state) => { - expect(false).toBeTruthy(); - }); - it('toBeGreaterThan', async (state) => { - expect(2).toBeGreaterThan(3); - }); - it('toBeNull', async (state) => { - expect(2).toBeNull(); - }); - it('toContain', async (state) => { - expect('asdf').toContain('e'); - }); - it('not.toContain', async (state) => { - expect('asdf').not.toContain('a'); - }); - it('toEqual', async (state) => { - expect([1,2,3]).toEqual([1,2,3,4]); - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/hookfail.js b/utils/testrunner/examples/hookfail.js deleted file mode 100644 index 18922505fa3ba..0000000000000 --- a/utils/testrunner/examples/hookfail.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter} = require('..'); - -const runner = new TestRunner(); -const reporter = new Reporter(runner); -const expect = require('expect'); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - beforeAll(() => { - expect(false).toBeTruthy(); - }); - it('test', async () => { - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/hooktimeout.js b/utils/testrunner/examples/hooktimeout.js deleted file mode 100644 index cf57d6b00fc3c..0000000000000 --- a/utils/testrunner/examples/hooktimeout.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter} = require('..'); - -const runner = new TestRunner({ timeout: 100 }); -const reporter = new Reporter(runner); -const expect = require('expect'); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - beforeAll(async () => { - await new Promise(() => {}); - }); - it('something', async (state) => { - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/timeout.js b/utils/testrunner/examples/timeout.js deleted file mode 100644 index 0d451e971e889..0000000000000 --- a/utils/testrunner/examples/timeout.js +++ /dev/null @@ -1,32 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter} = require('..'); - -const runner = new TestRunner({ timeout: 100 }); -const reporter = new Reporter(runner); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - it('timeout', async (state) => { - await new Promise(() => {}); - }); -}); - -runner.run(); diff --git a/utils/testrunner/examples/unhandledpromiserejection.js b/utils/testrunner/examples/unhandledpromiserejection.js deleted file mode 100644 index 270bfcc422171..0000000000000 --- a/utils/testrunner/examples/unhandledpromiserejection.js +++ /dev/null @@ -1,35 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const {TestRunner, Reporter} = require('..'); - -const runner = new TestRunner(); -const reporter = new Reporter(runner); - -const {describe, xdescribe, fdescribe} = runner; -const {it, fit, xit} = runner; -const {beforeAll, beforeEach, afterAll, afterEach} = runner; - -describe('testsuite', () => { - it('failure', async (state) => { - Promise.reject(new Error('fail!')); - }); - it('slow', async () => { - await new Promise(x => setTimeout(x, 1000)); - }); -}); - -runner.run(); diff --git a/utils/testrunner/index.js b/utils/testrunner/index.js deleted file mode 100644 index 3bd964619a341..0000000000000 --- a/utils/testrunner/index.js +++ /dev/null @@ -1,20 +0,0 @@ -/** - * Copyright 2017 Google Inc. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -const TestRunner = require('./TestRunner'); -const Reporter = require('./Reporter'); - -module.exports = { TestRunner, Reporter}; diff --git a/utils/testrunner/package.json b/utils/testrunner/package.json deleted file mode 100644 index fb4a9a2e8b5f7..0000000000000 --- a/utils/testrunner/package.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "name": "@pptr/testrunner", - "version": "0.8.0", - "description": "Puppeteer testrunner", - "main": "index.js", - "directories": { - "example": "examples" - }, - "scripts": { - "test": "node test/test.js" - }, - "repository": { - "type": "git", - "url": "https://github.com/puppeteer/puppeteer/tree/master/utils/testrunner" - }, - "author": "The Chromium Authors", - "license": "Apache-2.0" -} diff --git a/utils/testrunner/test/test.js b/utils/testrunner/test/test.js deleted file mode 100644 index 1e3eaf90d7ff1..0000000000000 --- a/utils/testrunner/test/test.js +++ /dev/null @@ -1,15 +0,0 @@ -const {TestRunner, Reporter} = require('..'); - -const testRunner = new TestRunner(); -const expect = require('expect'); - -require('./testrunner.spec.js').addTests({testRunner, expect}); - -new Reporter(testRunner, { - verbose: process.argv.includes('--verbose'), - summary: true, - projectFolder: require('path').join(__dirname, '..'), - showSlowTests: 0, -}); -testRunner.run(); - diff --git a/utils/testrunner/test/testrunner.spec.js b/utils/testrunner/test/testrunner.spec.js deleted file mode 100644 index e1d47edac49e2..0000000000000 --- a/utils/testrunner/test/testrunner.spec.js +++ /dev/null @@ -1,610 +0,0 @@ -const {TestRunner} = require('..'); - -module.exports.addTests = function({testRunner, expect}) { - const {describe, fdescribe, xdescribe} = testRunner; - const {it, xit, fit} = testRunner; - - describe('TestRunner.it', () => { - it('should declare a test', async() => { - const t = new TestRunner(); - t.it('uno', () => {}); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('uno'); - expect(test.declaredMode).toBe('run'); - expect(test.location.filePath).toEqual(__filename); - expect(test.location.fileName).toEqual('testrunner.spec.js'); - expect(test.location.lineNumber).toBeTruthy(); - expect(test.location.columnNumber).toBeTruthy(); - }); - }); - - describe('TestRunner.xit', () => { - it('should declare a skipped test', async() => { - const t = new TestRunner(); - t.xit('uno', () => {}); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('uno'); - expect(test.declaredMode).toBe('skip'); - }); - }); - - describe('TestRunner.fit', () => { - it('should declare a focused test', async() => { - const t = new TestRunner(); - t.fit('uno', () => {}); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('uno'); - expect(test.declaredMode).toBe('focus'); - }); - }); - - describe('TestRunner.describe', () => { - it('should declare a suite', async() => { - const t = new TestRunner(); - t.describe('suite', () => { - t.it('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.name).toBe('uno'); - expect(test.fullName).toBe('suite uno'); - expect(test.declaredMode).toBe('run'); - expect(test.suite.name).toBe('suite'); - expect(test.suite.fullName).toBe('suite'); - expect(test.suite.declaredMode).toBe('run'); - }); - }); - - describe('TestRunner.xdescribe', () => { - it('should declare a skipped suite', async() => { - const t = new TestRunner(); - t.xdescribe('suite', () => { - t.it('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.declaredMode).toBe('skip'); - expect(test.suite.declaredMode).toBe('skip'); - }); - it('focused tests inside a skipped suite are considered skipped', async() => { - const t = new TestRunner(); - t.xdescribe('suite', () => { - t.fit('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.declaredMode).toBe('skip'); - expect(test.suite.declaredMode).toBe('skip'); - }); - }); - - describe('TestRunner.fdescribe', () => { - it('should declare a focused suite', async() => { - const t = new TestRunner(); - t.fdescribe('suite', () => { - t.it('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.declaredMode).toBe('run'); - expect(test.suite.declaredMode).toBe('focus'); - }); - it('skipped tests inside a focused suite should stay skipped', async() => { - const t = new TestRunner(); - t.fdescribe('suite', () => { - t.xit('uno', () => {}); - }); - expect(t.tests().length).toBe(1); - const test = t.tests()[0]; - expect(test.declaredMode).toBe('skip'); - expect(test.suite.declaredMode).toBe('focus'); - }); - it('should run all "run" tests inside a focused suite', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - t.fdescribe('suite1', () => { - t.it('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.it('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('2,3'); - }); - it('should run only "focus" tests inside a focused suite', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - t.fdescribe('suite1', () => { - t.fit('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.it('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('2'); - }); - it('should run both "run" tests in focused suite and non-descendant focus tests', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - t.fdescribe('suite1', () => { - t.it('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.fit('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('2,3,4'); - }); - }); - - describe('TestRunner hooks', () => { - it('should run all hooks in proper order', async() => { - const log = []; - const t = new TestRunner(); - t.beforeAll(() => log.push('root:beforeAll')); - t.beforeEach(() => log.push('root:beforeEach')); - t.it('uno', () => log.push('test #1')); - t.describe('suite1', () => { - t.beforeAll(() => log.push('suite:beforeAll')); - t.beforeEach(() => log.push('suite:beforeEach')); - t.it('dos', () => log.push('test #2')); - t.it('tres', () => log.push('test #3')); - t.afterEach(() => log.push('suite:afterEach')); - t.afterAll(() => log.push('suite:afterAll')); - }); - t.it('cuatro', () => log.push('test #4')); - t.afterEach(() => log.push('root:afterEach')); - t.afterAll(() => log.push('root:afterAll')); - await t.run(); - expect(log).toEqual([ - 'root:beforeAll', - 'root:beforeEach', - 'test #1', - 'root:afterEach', - - 'suite:beforeAll', - - 'root:beforeEach', - 'suite:beforeEach', - 'test #2', - 'suite:afterEach', - 'root:afterEach', - - 'root:beforeEach', - 'suite:beforeEach', - 'test #3', - 'suite:afterEach', - 'root:afterEach', - - 'suite:afterAll', - - 'root:beforeEach', - 'test #4', - 'root:afterEach', - - 'root:afterAll', - ]); - }); - it('should have the same state object in hooks and test', async() => { - const states = []; - const t = new TestRunner(); - t.beforeEach(state => states.push(state)); - t.afterEach(state => states.push(state)); - t.beforeAll(state => states.push(state)); - t.afterAll(state => states.push(state)); - t.it('uno', state => states.push(state)); - await t.run(); - expect(states.length).toBe(5); - for (let i = 1; i < states.length; ++i) - expect(states[i]).toBe(states[0]); - }); - it('should unwind hooks properly when terminated', async() => { - const log = []; - const t = new TestRunner({timeout: 10000}); - t.beforeAll(() => log.push('beforeAll')); - t.beforeEach(() => log.push('beforeEach')); - t.afterEach(() => log.push('afterEach')); - t.afterAll(() => log.push('afterAll')); - t.it('uno', () => { - log.push('terminating...'); - t.terminate(); - }); - await t.run(); - - expect(log).toEqual([ - 'beforeAll', - 'beforeEach', - 'terminating...', - 'afterEach', - 'afterAll', - ]); - }); - it('should unwind hooks properly when crashed', async() => { - const log = []; - const t = new TestRunner({timeout: 10000}); - t.beforeAll(() => log.push('root beforeAll')); - t.beforeEach(() => log.push('root beforeEach')); - t.describe('suite', () => { - t.beforeAll(() => log.push('suite beforeAll')); - t.beforeEach(() => log.push('suite beforeEach')); - t.it('uno', () => log.push('uno')); - t.afterEach(() => { - log.push('CRASH >> suite afterEach'); - throw new Error('crash!'); - }); - t.afterAll(() => log.push('suite afterAll')); - }); - t.afterEach(() => log.push('root afterEach')); - t.afterAll(() => log.push('root afterAll')); - await t.run(); - - expect(log).toEqual([ - 'root beforeAll', - 'suite beforeAll', - 'root beforeEach', - 'suite beforeEach', - 'uno', - 'CRASH >> suite afterEach', - 'root afterEach', - 'suite afterAll', - 'root afterAll' - ]); - }); - }); - - describe('TestRunner.run', () => { - it('should run a test', async() => { - const t = new TestRunner(); - let ran = false; - t.it('uno', () => ran = true); - await t.run(); - expect(ran).toBe(true); - }); - it('should run tests if some fail', async() => { - const t = new TestRunner(); - const log = []; - t.it('uno', () => log.push(1)); - t.it('dos', () => { throw new Error('bad'); }); - t.it('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('1,3'); - }); - it('should run tests if some timeout', async() => { - const t = new TestRunner({timeout: 1}); - const log = []; - t.it('uno', () => log.push(1)); - t.it('dos', async() => new Promise(() => {})); - t.it('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('1,3'); - }); - it('should break on first failure if configured so', async() => { - const log = []; - const t = new TestRunner({breakOnFailure: true}); - t.it('test#1', () => log.push('test#1')); - t.it('test#2', () => log.push('test#2')); - t.it('test#3', () => { throw new Error('crash'); }); - t.it('test#4', () => log.push('test#4')); - await t.run(); - expect(log).toEqual([ - 'test#1', - 'test#2', - ]); - }); - it('should pass a state and a test as a test parameters', async() => { - const log = []; - const t = new TestRunner(); - t.beforeEach(state => state.FOO = 42); - t.it('uno', (state, test) => { - log.push('state.FOO=' + state.FOO); - log.push('test=' + test.name); - }); - await t.run(); - expect(log.join()).toBe('state.FOO=42,test=uno'); - }); - it('should run async test', async() => { - const t = new TestRunner(); - let ran = false; - t.it('uno', async() => { - await new Promise(x => setTimeout(x, 10)); - ran = true; - }); - await t.run(); - expect(ran).toBe(true); - }); - it('should run async tests in order of their declaration', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', async() => { - await new Promise(x => setTimeout(x, 30)); - log.push(1); - }); - t.it('dos', async() => { - await new Promise(x => setTimeout(x, 20)); - log.push(2); - }); - t.it('tres', async() => { - await new Promise(x => setTimeout(x, 10)); - log.push(3); - }); - await t.run(); - expect(log.join()).toBe('1,2,3'); - }); - it('should run multiple tests', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - t.it('dos', () => log.push(2)); - await t.run(); - expect(log.join()).toBe('1,2'); - }); - it('should NOT run a skipped test', async() => { - const t = new TestRunner(); - let ran = false; - t.xit('uno', () => ran = true); - await t.run(); - expect(ran).toBe(false); - }); - it('should run ONLY non-skipped tests', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - t.xit('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('1,3'); - }); - it('should run ONLY focused tests', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - t.xit('dos', () => log.push(2)); - t.fit('tres', () => log.push(3)); - await t.run(); - expect(log.join()).toBe('3'); - }); - it('should run tests in order of their declaration', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - t.describe('suite1', () => { - t.it('dos', () => log.push(2)); - t.it('tres', () => log.push(3)); - }); - t.it('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('1,2,3,4'); - }); - it('should support async test suites', async() => { - const log = []; - const t = new TestRunner(); - t.it('uno', () => log.push(1)); - await t.describe('suite1', async() => { - await Promise.resolve(); - t.it('dos', () => log.push(2)); - await Promise.resolve(); - t.it('tres', () => log.push(3)); - }); - t.it('cuatro', () => log.push(4)); - await t.run(); - expect(log.join()).toBe('1,2,3,4'); - }); - }); - - describe('TestRunner.run result', () => { - it('should return OK if all tests pass', async() => { - const t = new TestRunner(); - t.it('uno', () => {}); - const result = await t.run(); - expect(result.result).toBe('ok'); - }); - it('should return FAIL if at least one test fails', async() => { - const t = new TestRunner(); - t.it('uno', () => { throw new Error('woof'); }); - const result = await t.run(); - expect(result.result).toBe('failed'); - }); - it('should return FAIL if at least one test times out', async() => { - const t = new TestRunner({timeout: 1}); - t.it('uno', async() => new Promise(() => {})); - const result = await t.run(); - expect(result.result).toBe('failed'); - }); - it('should return TERMINATED if it was terminated', async() => { - const t = new TestRunner({timeout: 1}); - t.it('uno', async() => new Promise(() => {})); - const [result] = await Promise.all([ - t.run(), - t.terminate(), - ]); - expect(result.result).toBe('terminated'); - }); - it('should return CRASHED if it crashed', async() => { - const t = new TestRunner({timeout: 1}); - t.it('uno', async() => new Promise(() => {})); - t.afterAll(() => { throw new Error('woof');}); - const result = await t.run(); - expect(result.result).toBe('crashed'); - }); - }); - - describe('TestRunner parallel', () => { - it('should run tests in parallel', async() => { - const log = []; - const t = new TestRunner({parallel: 2}); - t.it('uno', async state => { - log.push(`Worker #${state.parallelIndex} Starting: UNO`); - await Promise.resolve(); - log.push(`Worker #${state.parallelIndex} Ending: UNO`); - }); - t.it('dos', async state => { - log.push(`Worker #${state.parallelIndex} Starting: DOS`); - await Promise.resolve(); - log.push(`Worker #${state.parallelIndex} Ending: DOS`); - }); - await t.run(); - expect(log).toEqual([ - 'Worker #0 Starting: UNO', - 'Worker #1 Starting: DOS', - 'Worker #0 Ending: UNO', - 'Worker #1 Ending: DOS', - ]); - }); - }); - - describe('TestRunner.hasFocusedTestsOrSuites', () => { - it('should work', () => { - const t = new TestRunner(); - t.it('uno', () => {}); - expect(t.hasFocusedTestsOrSuites()).toBe(false); - }); - it('should work #2', () => { - const t = new TestRunner(); - t.fit('uno', () => {}); - expect(t.hasFocusedTestsOrSuites()).toBe(true); - }); - it('should work #3', () => { - const t = new TestRunner(); - t.describe('suite #1', () => { - t.fdescribe('suite #2', () => { - t.describe('suite #3', () => { - t.it('uno', () => {}); - }); - }); - }); - expect(t.hasFocusedTestsOrSuites()).toBe(true); - }); - }); - - describe('TestRunner.passedTests', () => { - it('should work', async() => { - const t = new TestRunner(); - t.it('uno', () => {}); - await t.run(); - expect(t.failedTests().length).toBe(0); - expect(t.skippedTests().length).toBe(0); - expect(t.passedTests().length).toBe(1); - const [test] = t.passedTests(); - expect(test.result).toBe('ok'); - }); - }); - - describe('TestRunner.failedTests', () => { - it('should work for both throwing and timeouting tests', async() => { - const t = new TestRunner({timeout: 1}); - t.it('uno', () => { throw new Error('boo');}); - t.it('dos', () => new Promise(() => {})); - await t.run(); - expect(t.skippedTests().length).toBe(0); - expect(t.passedTests().length).toBe(0); - expect(t.failedTests().length).toBe(2); - const [test1, test2] = t.failedTests(); - expect(test1.result).toBe('failed'); - expect(test2.result).toBe('timedout'); - }); - it('should report crashed tests', async() => { - const t = new TestRunner(); - t.beforeEach(() => { throw new Error('woof');}); - t.it('uno', () => {}); - await t.run(); - expect(t.failedTests().length).toBe(1); - expect(t.failedTests()[0].result).toBe('crashed'); - }); - }); - - describe('TestRunner.skippedTests', () => { - it('should work for both throwing and timeouting tests', async() => { - const t = new TestRunner({timeout: 1}); - t.xit('uno', () => { throw new Error('boo');}); - await t.run(); - expect(t.skippedTests().length).toBe(1); - expect(t.passedTests().length).toBe(0); - expect(t.failedTests().length).toBe(0); - const [test] = t.skippedTests(); - expect(test.result).toBe('skipped'); - }); - }); - - describe('Test.result', () => { - it('should return OK', async() => { - const t = new TestRunner(); - t.it('uno', () => {}); - await t.run(); - expect(t.tests()[0].result).toBe('ok'); - }); - it('should return TIMEDOUT', async() => { - const t = new TestRunner({timeout: 1}); - t.it('uno', async() => new Promise(() => {})); - await t.run(); - expect(t.tests()[0].result).toBe('timedout'); - }); - it('should return SKIPPED', async() => { - const t = new TestRunner(); - t.xit('uno', () => {}); - await t.run(); - expect(t.tests()[0].result).toBe('skipped'); - }); - it('should return FAILED', async() => { - const t = new TestRunner(); - t.it('uno', async() => Promise.reject('woof')); - await t.run(); - expect(t.tests()[0].result).toBe('failed'); - }); - it('should return TERMINATED', async() => { - const t = new TestRunner(); - t.it('uno', async() => t.terminate()); - await t.run(); - expect(t.tests()[0].result).toBe('terminated'); - }); - it('should return CRASHED', async() => { - const t = new TestRunner(); - t.it('uno', () => {}); - t.afterEach(() => {throw new Error('foo');}); - await t.run(); - expect(t.tests()[0].result).toBe('crashed'); - }); - }); - - describe('TestRunner Events', () => { - it('should emit events in proper order', async() => { - const log = []; - const t = new TestRunner(); - t.beforeAll(() => log.push('beforeAll')); - t.beforeEach(() => log.push('beforeEach')); - t.it('test#1', () => log.push('test#1')); - t.afterEach(() => log.push('afterEach')); - t.afterAll(() => log.push('afterAll')); - t.on('started', () => log.push('E:started')); - t.on('teststarted', () => log.push('E:teststarted')); - t.on('testfinished', () => log.push('E:testfinished')); - t.on('finished', () => log.push('E:finished')); - await t.run(); - expect(log).toEqual([ - 'E:started', - 'beforeAll', - 'E:teststarted', - 'beforeEach', - 'test#1', - 'afterEach', - 'E:testfinished', - 'afterAll', - 'E:finished', - ]); - }); - it('should emit finish event with result', async() => { - const t = new TestRunner(); - const [result] = await Promise.all([ - new Promise(x => t.once('finished', x)), - t.run(), - ]); - expect(result.result).toBe('ok'); - }); - }); -}; -