test(perf): add benchmark for jest runner (#2618)

* Add a git submodule for https://github.com/GoogleChrome/lighthouse, a big open source project that uses jest * Add functionality to run performance tests without installing stryker explicitly in the project * Add functionality to install dependencies with `yarn` instead of `npm` (used for lighthouse)
stryker-mutator · Nov 26, 2020 · 5964d55 · 5964d55
1 parent d11d4cc
commit 5964d55
Show file tree

Hide file tree

Showing 14 changed files with 1,548 additions and 1,436 deletions.
diff --git a/.github/workflows/performance.yml b/.github/workflows/performance.yml
@@ -24,3 +24,8 @@ jobs:
       run: npm run perf
       env:
         PERF_TEST_GLOB_PATTERN: ${{ github.event.inputs.PERF_TEST_GLOB_PATTERN }}
+    - name: Store reports
+      uses: actions/upload-artifact@v2
+      with:
+        name: mutation-testing-reports
+        path: perf/test/*/reports/mutation
diff --git a/.gitmodules b/.gitmodules
@@ -1,3 +1,6 @@
 [submodule "perf/test/express"]
 	path = perf/test/express
 	url = https://github.com/expressjs/express.git
+[submodule "perf/test/lighthouse"]
+	path = perf/test/lighthouse
+	url = https://github.com/GoogleChrome/lighthouse.git
diff --git a/perf/config/express/package.json b/perf/config/express/package.json
diff --git a/perf/config/lighthouse/stryker.conf.json b/perf/config/lighthouse/stryker.conf.json
@@ -0,0 +1,24 @@
+{
+  "$schema": "../../../packages/core/schema/stryker-schema.json",
+  "packageManager": "yarn",
+  "reporters": [
+    "html",
+    "clear-text",
+    "progress",
+    "json"
+  ],
+  "jest": {
+    "config": {
+      "testMatch": ["**/lighthouse-core/test/audits/**/*-test.js"],
+      "setupFilesAfterEnv": ["./lighthouse-core/test/test-utils.js"],
+      "testEnvironment": "node",
+      "transform": {},
+      "collectCoverage": false
+    }
+  },
+  "mutate": [
+    "lighthouse-core/audits/**/*.js"
+  ],
+  "testRunner": "jest",
+  "coverageAnalysis": "off"
+}
diff --git a/perf/package.json b/perf/package.json
@@ -6,9 +6,9 @@
     "ts-node": "~8.10.2"
   },
   "scripts": {
+    "postinstall": "npm run merge-config && npm run bootstrap",
     "merge-config": "ts-node -p tasks/merge-config.ts",
-    "postinstall": "npm run merge-config && node ../node_modules/lerna/cli.js bootstrap --no-ci && npm run install-local-dependencies",
-    "install-local-dependencies": "ts-node -p tasks/install-local-dependencies.ts",
+    "bootstrap": "ts-node -p tasks/install.ts",
     "test": "ts-node tasks/run-perf-tests.ts"
   }
 }
diff --git a/perf/tasks/install-local-dependencies.ts b/perf/tasks/install-local-dependencies.ts
diff --git a/perf/tasks/install.ts b/perf/tasks/install.ts
@@ -0,0 +1,44 @@
+import * as fs from 'fs';
+import * as minimatch from 'minimatch';
+import * as path from 'path';
+import * as execa from 'execa';
+
+const testRootDir = path.resolve(__dirname, '..', 'test');
+
+installAll()
+  .then(() => console.log('Done'))
+  .catch((err) => {
+    console.error(err);
+    process.exit(1);
+  });
+
+/**
+ * Installs all packages under the "test" directory, while respecting their preferred package manager (yarn vs npm 🙄)
+ */
+async function installAll() {
+  const globPattern = process.env.PERF_TEST_GLOB_PATTERN || '*';
+  const testDirs = fs.readdirSync(testRootDir).filter((testDir) => minimatch(testDir, globPattern));
+  if (testDirs.length) {
+    console.log(`Installing ${testDirs.join(', ')} (matched with glob pattern "${globPattern}")`);
+  } else {
+    console.warn(`No tests match glob expression ${globPattern}`);
+  }
+  await Promise.all(testDirs.map(install));
+}
+
+async function install(testDir: string) {
+  const strykerConfig = require(`../test/${testDir}/stryker.conf`);
+  const packageManager: string | undefined = strykerConfig.packageManager;
+  let command = 'npm';
+  let args: string[] = [];
+  if (packageManager === 'yarn') {
+    command = 'yarn';
+    args.push('install', '--frozen-lockfile');
+  } else if(fs.existsSync(path.resolve(testRootDir, testDir, 'package-lock.json'))) {
+    args.push('ci');
+  } else {
+    args.push('install');
+  }
+  console.log(`[${testDir}] ${command} ${args.join(' ')}`);
+  await execa(command, args, { cwd: path.resolve(testRootDir, testDir) });
+}
diff --git a/perf/tasks/merge-config.ts b/perf/tasks/merge-config.ts
@@ -1,16 +1,16 @@
 import { promises as fs } from 'fs';
-import path = require("path");
+import path = require('path');
 
 const testRootDir = path.resolve(__dirname, '..', 'test');
 const configRootDir = path.resolve(__dirname, '..', 'config');
 
-mergeConfiguration().catch(error => {
+mergeConfiguration().catch((error) => {
   console.error(error);
   process.exitCode = 1;
 });
 
 async function mergeConfiguration() {
-  const testDirs = (await fs.readdir(testRootDir, { withFileTypes: true })).filter(testDir => testDir.isDirectory());
+  const testDirs = (await fs.readdir(testRootDir, { withFileTypes: true })).filter((testDir) => testDir.isDirectory());
   for await (const testDir of testDirs) {
     const configOverrideDir = path.resolve(configRootDir, testDir.name);
     try {
@@ -21,7 +21,7 @@ async function mergeConfiguration() {
         try {
           const overrides = require(overridePackageFileName);
           const original = require(path.resolve(testRootDir, testDir.name, 'package.json'));
-          await fs.writeFile(path.resolve(testRootDir, testDir.name, 'package.json'), JSON.stringify({ ...original, ...overrides }, null, 2))
+          await fs.writeFile(path.resolve(testRootDir, testDir.name, 'package.json'), JSON.stringify({ ...original, ...overrides }, null, 2));
         } catch {
           console.log(`Note: no overrides found at ${overridePackageFileName}`);
         }
@@ -31,7 +31,7 @@ async function mergeConfiguration() {
           console.log(`Note: no stryker.conf.json file ${overrideStrykerConfigFileName}`);
         }
       }
-      console.log(`✅ Merged config for ${testDir.name}`)
+      console.log(`✅ Merged config for ${testDir.name}`);
     } catch {
       console.log(`Note: no config override directory found at ${configOverrideDir}`);
     }

diff --git a/perf/tasks/run-perf-tests.ts b/perf/tasks/run-perf-tests.ts
@@ -9,16 +9,16 @@ const testRootDir = path.resolve(__dirname, '..', 'test');
 
 runPerfTests()
   .then(() => console.log('Done'))
-  .catch(err => {
+  .catch((err) => {
     console.error(err);
     process.exit(1);
   });
 
 async function runPerfTests() {
   const globPattern = process.env.PERF_TEST_GLOB_PATTERN || '*';
-  const testDirs = fs.readdirSync(testRootDir).filter(testDir => minimatch(testDir, globPattern));
+  const testDirs = fs.readdirSync(testRootDir).filter((testDir) => minimatch(testDir, globPattern));
   if (testDirs.length) {
-    console.log(`Running performance tests on ${testDirs.join(', ')} (matched with glob pattern "${globPattern}")`)
+    console.log(`Running performance tests on ${testDirs.join(', ')} (matched with glob pattern "${globPattern}")`);
   } else {
     console.warn(`No test files match glob expression ${globPattern}`);
   }
@@ -32,24 +32,37 @@ async function runPerfTests() {
 
 async function runTest(testDir: string) {
   console.time(testDir);
-  await npx(['stryker', 'run'], testDir).pipe(
-    throttleTime(60000),
-    tap(logMessage => console.timeLog(testDir, 'last log message: ', logMessage))
-  ).toPromise();
+  await runStryker(testDir)
+    .pipe(
+      throttleTime(60000),
+      tap((logMessage) => console.timeLog(testDir, 'last log message: ', logMessage))
+    )
+    .toPromise();
   console.timeEnd(testDir);
 }
 
-function npx(args: string[], testDir: string): Observable<string> {
+function runStryker(testDir: string): Observable<string> {
+  const strykerBin = require.resolve('../../packages/core/bin/stryker');
+  const args = [
+    'run',
+    '--plugins',
+    [
+      require.resolve('../../packages/mocha-runner'),
+      require.resolve('../../packages/karma-runner'),
+      require.resolve('../../packages/jest-runner'),
+      require.resolve('../../packages/jasmine-runner'),
+      require.resolve('../../packages/mocha-runner'),
+      require.resolve('../../packages/typescript-checker'),
+    ].join(','),
+  ];
   const currentTestDir = path.resolve(testRootDir, testDir);
-  console.log(`Exec ${testDir} npx ${args.join(' ')}`);
+  console.log(`(${testDir}) exec "${strykerBin} ${args.join(' ')}"`);
 
-  return new Observable(observer => {
-    const testProcess = execa('npx', args, { timeout: 0, cwd: currentTestDir, stdio: 'pipe' });
+  return new Observable((observer) => {
+    const testProcess = execa(strykerBin, args, { timeout: 0, cwd: currentTestDir, stdio: 'pipe' });
     let stderr = '';
-    testProcess.stderr?.on('data', chunk => stderr += chunk.toString());
-    testProcess.stdout?.on('data', chunk => observer.next(chunk.toString().trim()));
-    testProcess
-      .then(() => observer.complete())
-      .catch(error => observer.error(error));
+    testProcess.stderr?.on('data', (chunk) => (stderr += chunk.toString()));
+    testProcess.stdout?.on('data', (chunk) => observer.next(chunk.toString().trim()));
+    testProcess.then(() => observer.complete()).catch((error) => observer.error(error));
   });
 }
diff --git a/perf/test/angular-cli/angular.json b/perf/test/angular-cli/angular.json
@@ -67,8 +67,7 @@
         "serve": {
           "builder": "@angular-devkit/build-angular:dev-server",
           "options": {
-            "browserTarget": "FEWebwinkel:build",
-            "proxyConfig": "proxy.conf.json"
+            "browserTarget": "FEWebwinkel:build"
           },
           "configurations": {
             "production": {