From 926a28684282aeec37680bbc52a66973b8055f54 Mon Sep 17 00:00:00 2001 From: Christian Bromann Date: Tue, 29 Aug 2023 19:50:25 -0700 Subject: [PATCH] test: replace Karma with Webdriver.IO (#17126) * test: replace Karma with Webdriver.IO The current test framework for browser testing (Karma) is not maintained anymore and WebdriverIO provides a more modern stack that allows to test in different browser. This patch replaces these test frameworks. fixes: #17009 * update webdriverio deps * PR feedback * adjust tests * build eslint before running tests * make test file an esm file * revert more esm changes * make it work * remove return value * custom log dir for wdio tests * auto detect chromedriver * bump timeout, store logs * bump timeout again * update wdio deps * update wdio deps * set log level to trace * update wdio deps and unskip tests * no need to have this be an async test * update deps * make path spec file explicit * remove Chromedriver deps * removed wdio command --- .github/workflows/ci.yml | 8 +- .gitignore | 2 + Makefile.js | 8 +- karma.conf.js | 125 ----------- lib/config/rule-validator.js | 3 +- package.json | 14 +- tests/lib/linter/linter.js | 14 +- wdio.conf.js | 387 +++++++++++++++++++++++++++++++++++ 8 files changed, 416 insertions(+), 145 deletions(-) delete mode 100644 karma.conf.js create mode 100644 wdio.conf.js diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 27dae4c1262..92e2a7eed35 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -75,6 +75,12 @@ jobs: - name: Install Packages run: npm install - name: Test - run: node Makefile karma + run: node Makefile wdio - name: Fuzz Test run: node Makefile fuzz + - uses: actions/upload-artifact@v3 + if: failure() + with: + name: logs + path: | + wdio-logs/*.log diff --git a/.gitignore b/.gitignore index 075a4d740c7..148181e0776 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,8 @@ test.js coverage/ build/ +logs +wdio-logs npm-debug.log yarn-error.log .pnpm-debug.log diff --git a/Makefile.js b/Makefile.js index 717cc785946..528fd5aa1e7 100644 --- a/Makefile.js +++ b/Makefile.js @@ -628,12 +628,10 @@ target.mocha = () => { } }; -target.karma = () => { +target.wdio = () => { echo("Running unit tests on browsers"); - target.webpack("production"); - - const lastReturn = exec(`${getBinFile("karma")} start karma.conf.js`); + const lastReturn = exec(`${getBinFile("wdio")} run wdio.conf.js`); if (lastReturn.code !== 0) { exit(1); @@ -643,7 +641,7 @@ target.karma = () => { target.test = function() { target.checkRuleFiles(); target.mocha(); - target.karma(); + target.wdio(); target.fuzz({ amount: 150, fuzzBrokenAutofixes: false }); target.checkLicenses(); }; diff --git a/karma.conf.js b/karma.conf.js deleted file mode 100644 index 606d13f88f6..00000000000 --- a/karma.conf.js +++ /dev/null @@ -1,125 +0,0 @@ -"use strict"; -const os = require("os"); -const NodePolyfillPlugin = require("node-polyfill-webpack-plugin"); - -if (os.platform === "linux" && os.arch() === "arm64") { - - // For arm64 architecture, install chromium-browser using "apt-get install chromium-browser" - process.env.CHROME_BIN = "/usr/bin/chromium-browser"; -} else { - process.env.CHROME_BIN = require("puppeteer").executablePath(); -} - -module.exports = function(config) { - config.set({ - - // base path that will be used to resolve all patterns (eg. files, exclude) - basePath: "", - - // next three sections allow console.log to work - client: { - captureConsole: true - }, - - browserConsoleLogOptions: { - terminal: true, - level: "log" - }, - - /* - * frameworks to use - * available frameworks: https://npmjs.org/browse/keyword/karma-adapter - */ - frameworks: ["mocha", "webpack"], - - - // list of files / patterns to load in the browser - files: [ - "tests/lib/linter/linter.js" - ], - - - // list of files to exclude - exclude: [ - ], - - - /* - * preprocess matching files before serving them to the browser - * available preprocessors: https://npmjs.org/browse/keyword/karma-preprocessor - */ - preprocessors: { - "tests/lib/linter/linter.js": ["webpack"] - }, - webpack: { - mode: "none", - plugins: [ - new NodePolyfillPlugin() - ], - resolve: { - alias: { - "../../../lib/linter$": "../../../build/eslint.js" - } - }, - stats: "errors-only" - }, - webpackMiddleware: { - logLevel: "error" - }, - - - /* - * test results reporter to use - * possible values: "dots", "progress" - * available reporters: https://npmjs.org/browse/keyword/karma-reporter - */ - reporters: ["mocha"], - - mochaReporter: { - output: "minimal" - }, - - // web server port - port: 9876, - - - // enable / disable colors in the output (reporters and logs) - colors: true, - - - /* - * level of logging - * possible values: config.LOG_DISABLE || config.LOG_ERROR || config.LOG_WARN || config.LOG_INFO || config.LOG_DEBUG - */ - logLevel: config.LOG_INFO, - - - // enable / disable watching file and executing tests whenever any file changes - autoWatch: false, - - - /* - * start these browsers - * available browser launchers: https://npmjs.org/browse/keyword/karma-launcher - */ - browsers: ["HeadlessChrome"], - customLaunchers: { - HeadlessChrome: { - base: "ChromeHeadless", - flags: ["--no-sandbox"] - } - }, - - /* - * Continuous Integration mode - * if true, Karma captures browsers, runs the tests and exits - */ - singleRun: true, - - /* - * Concurrency level - * how many browser should be started simultaneous - */ - concurrency: Infinity - }); -}; diff --git a/lib/config/rule-validator.js b/lib/config/rule-validator.js index 0b5858fb30f..eee5b40bd07 100644 --- a/lib/config/rule-validator.js +++ b/lib/config/rule-validator.js @@ -9,7 +9,8 @@ // Requirements //----------------------------------------------------------------------------- -const ajv = require("../shared/ajv")(); +const ajvImport = require("../shared/ajv"); +const ajv = ajvImport(); const { parseRuleId, getRuleFromConfig, diff --git a/package.json b/package.json index d52a2aef60c..4c22a4a94f0 100644 --- a/package.json +++ b/package.json @@ -101,6 +101,11 @@ "devDependencies": { "@babel/core": "^7.4.3", "@babel/preset-env": "^7.4.3", + "@wdio/browser-runner": "^8.14.6", + "@wdio/cli": "^8.14.6", + "@wdio/concise-reporter": "^8.14.0", + "@wdio/globals": "^8.14.6", + "@wdio/mocha-framework": "^8.14.0", "babel-loader": "^8.0.5", "c8": "^7.12.0", "chai": "^4.0.1", @@ -124,11 +129,6 @@ "glob": "^7.1.6", "got": "^11.8.3", "gray-matter": "^4.0.3", - "karma": "^6.1.1", - "karma-chrome-launcher": "^3.1.0", - "karma-mocha": "^2.0.1", - "karma-mocha-reporter": "^2.2.5", - "karma-webpack": "^5.0.0", "lint-staged": "^11.0.0", "load-perf": "^0.2.0", "markdownlint": "^0.25.1", @@ -148,12 +148,14 @@ "pirates": "^4.0.5", "progress": "^2.0.3", "proxyquire": "^2.0.1", - "puppeteer": "^13.7.0", "recast": "^0.20.4", "regenerator-runtime": "^0.13.2", + "rollup-plugin-node-polyfills": "^0.2.1", "semver": "^7.5.3", "shelljs": "^0.8.2", "sinon": "^11.0.0", + "vite-plugin-commonjs": "^0.8.2", + "webdriverio": "^8.14.6", "webpack": "^5.23.0", "webpack-cli": "^4.5.0", "yorkie": "^2.0.0" diff --git a/tests/lib/linter/linter.js b/tests/lib/linter/linter.js index f86e98ba8e0..65957f82a75 100644 --- a/tests/lib/linter/linter.js +++ b/tests/lib/linter/linter.js @@ -9,7 +9,7 @@ // Requirements //------------------------------------------------------------------------------ -const assert = require("chai").assert, +const { assert } = require("chai"), sinon = require("sinon"), espree = require("espree"), esprima = require("esprima"), @@ -7263,12 +7263,12 @@ var a = "test2"; it("should have file path passed to it", () => { const code = "/* this is code */"; - const parseSpy = sinon.spy(testParsers.stubParser, "parse"); + const parseSpy = { parse: sinon.spy() }; - linter.defineParser("stub-parser", testParsers.stubParser); + linter.defineParser("stub-parser", parseSpy); linter.verify(code, { parser: "stub-parser" }, filename, true); - sinon.assert.calledWithMatch(parseSpy, "", { filePath: filename }); + sinon.assert.calledWithMatch(parseSpy.parse, "", { filePath: filename }); }); it("should not report an error when JSX code contains a spread operator and JSX is enabled", () => { @@ -8068,16 +8068,16 @@ describe("Linter with FlatConfigArray", () => { it("should have file path passed to it", () => { const code = "/* this is code */"; - const parseSpy = sinon.spy(testParsers.stubParser, "parse"); + const parseSpy = { parse: sinon.spy() }; const config = { languageOptions: { - parser: testParsers.stubParser + parser: parseSpy } }; linter.verify(code, config, filename, true); - sinon.assert.calledWithMatch(parseSpy, "", { filePath: filename }); + sinon.assert.calledWithMatch(parseSpy.parse, "", { filePath: filename }); }); it("should not report an error when JSX code contains a spread operator and JSX is enabled", () => { diff --git a/wdio.conf.js b/wdio.conf.js new file mode 100644 index 00000000000..f32d757133e --- /dev/null +++ b/wdio.conf.js @@ -0,0 +1,387 @@ +"use strict"; + +const path = require("path"); +const commonjs = require("vite-plugin-commonjs").default; + +exports.config = { + + /* + * + * ==================== + * Runner Configuration + * ==================== + * WebdriverIO supports running e2e tests as well as unit and component tests. + */ + runner: ["browser", { + viteConfig: { + resolve: { + alias: { + util: "rollup-plugin-node-polyfills/polyfills/util", + path: "rollup-plugin-node-polyfills/polyfills/path", + assert: "rollup-plugin-node-polyfills/polyfills/assert" + } + }, + plugins: [ + commonjs(), + { + name: "wdio:import-fix", + enforce: "pre", + transform(source, id) { + if (!id.endsWith("/tests/lib/linter/linter.js")) { + return source; + } + + return source.replace( + 'const { Linter } = require("../../../lib/linter");', + 'const { Linter } = require("../../../build/eslint");\n' + + 'process.cwd = () => "/";' + ); + } + } + ] + } + }], + + /* + * + * ================== + * Specify Test Files + * ================== + * Define which test specs should run. The pattern is relative to the directory + * of the configuration file being run. + * + * The specs are defined as an array of spec files (optionally using wildcards + * that will be expanded). The test for each spec file will be run in a separate + * worker process. In order to have a group of spec files run in the same worker + * process simply enclose them in an array within the specs array. + * + * If you are calling `wdio` from an NPM script (see https://docs.npmjs.com/cli/run-script), + * then the current working directory is where your `package.json` resides, so `wdio` + * will be called from there. + * + */ + specs: [ + path.join(__dirname, "tests", "lib", "linter", "linter.js") + ], + + // Patterns to exclude. + exclude: [], + + /* + * + * ============ + * Capabilities + * ============ + * Define your capabilities here. WebdriverIO can run multiple capabilities at the same + * time. Depending on the number of capabilities, WebdriverIO launches several test + * sessions. Within your capabilities you can overwrite the spec and exclude options in + * order to group specific specs to a specific capability. + * + * First, you can define how many instances should be started at the same time. Let"s + * say you have 3 different capabilities (Chrome, Firefox, and Safari) and you have + * set maxInstances to 1; wdio will spawn 3 processes. Therefore, if you have 10 spec + * files and you set maxInstances to 10, all spec files will get tested at the same time + * and 30 processes will get spawned. The property handles how many capabilities + * from the same test should run tests. + * + */ + maxInstances: 10, + + /* + * + * If you have trouble getting all important capabilities together, check out the + * Sauce Labs platform configurator - a great tool to configure your capabilities: + * https://saucelabs.com/platform/platform-configurator + * + */ + capabilities: [{ + browserName: "chrome", + "goog:chromeOptions": { + args: process.env.CI ? ["headless", "disable-gpu"] : [] + } + }], + + /* + * + * =================== + * Test Configurations + * =================== + * Define all options that are relevant for the WebdriverIO instance here + * + * Level of logging verbosity: trace | debug | info | warn | error | silent + */ + logLevel: "trace", + outputDir: "./wdio-logs", + + /* + * + * Set specific log levels per logger + * loggers: + * - webdriver, webdriverio + * - @wdio/browserstack-service, @wdio/devtools-service, @wdio/sauce-service + * - @wdio/mocha-framework, @wdio/jasmine-framework + * - @wdio/local-runner + * - @wdio/sumologic-reporter + * - @wdio/cli, @wdio/config, @wdio/utils + * Level of logging verbosity: trace | debug | info | warn | error | silent + * logLevels: { + * webdriver: 'info', + * '@wdio/appium-service': 'info' + * }, + * + * If you only want to run your tests until a specific amount of tests have failed use + * bail (default is 0 - don't bail, run all tests). + */ + bail: 0, + + /* + * + * Set a base URL in order to shorten url command calls. If your `url` parameter starts + * with `/`, the base url gets prepended, not including the path portion of your baseUrl. + * If your `url` parameter starts without a scheme or `/` (like `some/path`), the base url + * gets prepended directly. + */ + baseUrl: "", + + /* + * + * Default timeout for all waitFor* commands. + */ + waitforTimeout: 10000, + + /* + * + * Default timeout in milliseconds for request + * if browser driver or grid doesn't send response + */ + connectionRetryTimeout: 120000, + + /* + * + * Default request retries count + */ + connectionRetryCount: 3, + + /* + * Framework you want to run your specs with. + * The following are supported: Mocha, Jasmine, and Cucumber + * see also: https://webdriver.io/docs/frameworks + * + * Make sure you have the wdio adapter package for the specific framework installed + * before running any tests. + */ + framework: "mocha", + + /* + * + * The number of times to retry the entire specfile when it fails as a whole + * specFileRetries: 1, + * + * Delay in seconds between the spec file retry attempts + * specFileRetriesDelay: 0, + * + * Whether or not retried specfiles should be retried immediately or deferred to the end of the queue + * specFileRetriesDeferred: false, + * + * Test reporter for stdout. + * The only one supported by default is 'dot' + * see also: https://webdriver.io/docs/dot-reporter + */ + reporters: ["concise"], + + /* + * + * Options to be passed to Mocha. + * See the full list at http://mochajs.org/ + */ + mochaOpts: { + ui: "bdd", + timeout: 5 * 60 * 1000, // 5min + grep: "@skipWeb", + invert: true + } + + /* + * + * ===== + * Hooks + * ===== + * WebdriverIO provides several hooks you can use to interfere with the test process in order to enhance + * it and to build services around it. You can either apply a single function or an array of + * methods to it. If one of them returns with a promise, WebdriverIO will wait until that promise got + * resolved to continue. + */ + /** + * Gets executed once before all workers get launched. + * @param {Object} config wdio configuration object + * @param {Array} capabilities list of capabilities details + */ + /* + * onPrepare: function (config, capabilities) { + * }, + */ + /** + * Gets executed before a worker process is spawned and can be used to initialise specific service + * for that worker as well as modify runtime environments in an async fashion. + * @param {string} cid capability id (e.g 0-0) + * @param {Object} caps object containing capabilities for session that will be spawn in the worker + * @param {Object} specs specs to be run in the worker process + * @param {Object} args object that will be merged with the main configuration once worker is initialized + * @param {Object} execArgv list of string arguments passed to the worker process + */ + /* + * onWorkerStart: function (cid, caps, specs, args, execArgv) { + * }, + */ + /** + * Gets executed just after a worker process has exited. + * @param {string} cid capability id (e.g 0-0) + * @param {number} exitCode 0 - success, 1 - fail + * @param {Object} specs specs to be run in the worker process + * @param {number} retries number of retries used + */ + /* + * onWorkerEnd: function (cid, exitCode, specs, retries) { + * }, + */ + /** + * Gets executed just before initialising the webdriver session and test framework. It allows you + * to manipulate configurations depending on the capability or spec. + * @param {Object} config wdio configuration object + * @param {Array} capabilities list of capabilities details + * @param {Array} specs List of spec file paths that are to be run + * @param {string} cid worker id (e.g. 0-0) + */ + /* + * beforeSession: function (config, capabilities, specs, cid) { + * }, + */ + /** + * Gets executed before test execution begins. At this point you can access to all global + * variables like `browser`. It is the perfect place to define custom commands. + * @param {Array} capabilities list of capabilities details + * @param {Array} specs List of spec file paths that are to be run + * @param {Object} browser instance of created browser/device session + */ + /* + * before: function (capabilities, specs) { + * }, + */ + /** + * Runs before a WebdriverIO command gets executed. + * @param {string} commandName hook command name + * @param {Array} args arguments that command would receive + */ + /* + * beforeCommand: function (commandName, args) { + * }, + */ + /** + * Hook that gets executed before the suite starts + * @param {Object} suite suite details + */ + /* + * beforeSuite: function (suite) { + * }, + */ + /** + * Function to be executed before a test (in Mocha/Jasmine) starts. + */ + /* + * beforeTest: function (test, context) { + * }, + */ + /** + * Hook that gets executed _before_ a hook within the suite starts (e.g. runs before calling + * beforeEach in Mocha) + */ + /* + * beforeHook: function (test, context) { + * }, + */ + /** + * Hook that gets executed _after_ a hook within the suite starts (e.g. runs after calling + * afterEach in Mocha) + */ + /* + * afterHook: function (test, context, { error, result, duration, passed, retries }) { + * }, + */ + /** + * Function to be executed after a test (in Mocha/Jasmine only) + * @param {Object} test test object + * @param {Object} context scope object the test was executed with + * @param {Error} result.error error object in case the test fails, otherwise `undefined` + * @param {any} result.result return object of test function + * @param {number} result.duration duration of test + * @param {boolean} result.passed true if test has passed, otherwise false + * @param {Object} result.retries informations to spec related retries, e.g. `{ attempts: 0, limit: 0 }` + */ + /* + * afterTest: function(test, context, { error, result, duration, passed, retries }) { + * }, + */ + + + /** + * Hook that gets executed after the suite has ended + * @param {Object} suite suite details + */ + /* + * afterSuite: function (suite) { + * }, + */ + /** + * Runs after a WebdriverIO command gets executed + * @param {string} commandName hook command name + * @param {Array} args arguments that command would receive + * @param {number} result 0 - command success, 1 - command error + * @param {Object} error error object if any + */ + /* + * afterCommand: function (commandName, args, result, error) { + * }, + */ + /** + * Gets executed after all tests are done. You still have access to all global variables from + * the test. + * @param {number} result 0 - test pass, 1 - test fail + * @param {Array} capabilities list of capabilities details + * @param {Array} specs List of spec file paths that ran + */ + /* + * after: function (result, capabilities, specs) { + * }, + */ + /** + * Gets executed right after terminating the webdriver session. + * @param {Object} config wdio configuration object + * @param {Array} capabilities list of capabilities details + * @param {Array} specs List of spec file paths that ran + */ + /* + * afterSession: function (config, capabilities, specs) { + * }, + */ + /** + * Gets executed after all workers got shut down and the process is about to exit. An error + * thrown in the onComplete hook will result in the test run failing. + * @param {Object} exitCode 0 - success, 1 - fail + * @param {Object} config wdio configuration object + * @param {Array} capabilities list of capabilities details + * @param {Object} results object containing test results + */ + /* + * onComplete: function(exitCode, config, capabilities, results) { + * }, + */ + /** + * Gets executed when a refresh happens. + * @param {string} oldSessionId session ID of the old session + * @param {string} newSessionId session ID of the new session + */ + /* + * onReload: function(oldSessionId, newSessionId) { + * } + */ +};