Add benchmarks for runtime with many files

.github/workflows/ci.yml

@@ -19,6 +19,7 @@ jobs:
               run: npm install
             - name: Run tests
               run: npm test
+              timeout-minutes: 20
             - name: Coveralls
               uses: coverallsapp/github-action@master
               with:

benchmarks/measure.js

@@ -0,0 +1,39 @@
+'use strict';
+
+const os = require('os');
+const { performance } = require('perf_hooks');
+const times = require('ramda/src/times');
+const median = require('ramda/src/median');
+const map = require('ramda/src/map');
+const prop = require('ramda/src/prop');
+
+const [ { speed: cpuSpeed } ] = os.cpus();
+
+function clearRequireCache() {
+    Object.keys(require.cache).forEach(function (key) {
+        delete require.cache[key];
+    });
+}
+
+function runBenchmark(fn, count) {
+    const results = [];
+
+    times(() => {
+        const startTime = performance.now();
+        const startMemory = process.memoryUsage().rss;
+        fn();
+        const endTime = performance.now();
+        const endMemory = process.memoryUsage().rss;
+        const duration = endTime - startTime;
+        const memory = endMemory - startMemory;
+
+        results.push({ duration, memory });
+    }, count);
+
+    const medianDuration = median(map(prop('duration'), results));
+    const medianMemory = median(map(prop('memory'), results));
+
+    return { medianDuration, medianMemory };
+}
+
+module.exports = { runBenchmark, clearRequireCache, cpuSpeed };

benchmarks/runtime.bench.js

@@ -0,0 +1,106 @@
+'use strict';
+
+const { expect } = require('chai');
+const { Linter } = require('eslint');
+const times = require('ramda/src/times');
+const toPairs = require('ramda/src/toPairs');
+const fromPairs = require('ramda/src/fromPairs');
+const { runBenchmark, cpuSpeed } = require('./measure');
+const mochaPlugin = require('../');
+
+const recommendedRules = mochaPlugin.configs.recommended.rules;
+
+function lintManyFilesWithAllRecommendedRules({ numberOfFiles }) {
+    const linter = new Linter();
+
+    linter.defineRules(mochaPlugin.rules);
+
+    const config = {
+        rules: fromPairs(toPairs(recommendedRules).map(([ ruleName, ruleSettings ]) => {
+            const [ , ruleNameWithoutPrefix ] = ruleName.split('/');
+
+            return [ ruleNameWithoutPrefix, ruleSettings ];
+        })),
+        parserOptions: {
+            ecmaVersion: 2018
+        }
+    };
+
+    times((index) => {
+        const codeToLint = `
+            'use strict';
+            const assert = require('assert');
+            const sinon = require('sinon');
+            const sut = require('./sut');
+
+            describe('SUT ${index}', function () {
+                let fooStub;
+
+                before(() => {
+                    fooStub = sinon.stub(sut, 'foo');
+                });
+
+                after(() => {
+                    fooStub.restore();
+                });
+
+                beforeEach(function (done) {
+                    done();
+                });
+
+                afterEach(() => {
+                    fooStub.reset();
+                });
+
+                it('should work', async function () {
+                    const bar = {};
+
+                    await sut(bar);
+
+                    assert(fooStub.callCount === 42);
+                });
+
+                describe('nested suite', async () => {
+                    beforeEach(async function () {
+                        await sleep(200);
+                    });
+
+                    xit('doesn’t work yet', function () {
+                        sut();
+                    });
+                });
+
+                context('more context', function () {
+                    it.only('only here it works', () => {
+                        sut();
+                        assert(true);
+                    });
+                });
+            });
+        `;
+
+        linter.verify(codeToLint, config);
+    }, numberOfFiles);
+}
+
+describe('runtime', () => {
+    it('should not take longer as the defined budget to lint many files with the recommended config', () => {
+        const budget = 80000000 / cpuSpeed;
+
+        const { medianDuration } = runBenchmark(() => {
+            lintManyFilesWithAllRecommendedRules({ numberOfFiles: 350 });
+        }, 5);
+
+        expect(medianDuration).to.be.below(budget);
+    });
+
+    it('should not consume more memory as the defined budget to lint many files with the recommended config', () => {
+        const budget = 3000000;
+
+        const { medianMemory } = runBenchmark(() => {
+            lintManyFilesWithAllRecommendedRules({ numberOfFiles: 350 });
+        }, 5);
+
+        expect(medianMemory).to.be.below(budget);
+    });
+});

benchmarks/startup.bench.js

@@ -1,41 +1,7 @@
 'use strict';
 
-const os = require('os');
 const { expect } = require('chai');
-const { performance } = require('perf_hooks');
-const times = require('ramda/src/times');
-const median = require('ramda/src/median');
-const map = require('ramda/src/map');
-const prop = require('ramda/src/prop');
-
-const [ { speed: cpuSpeed } ] = os.cpus();
-
-function clearRequireCache() {
-    Object.keys(require.cache).forEach(function (key) {
-        delete require.cache[key];
-    });
-}
-
-function runBenchmark(fn, count) {
-    const results = [];
-
-    times(() => {
-        const startTime = performance.now();
-        const startMemory = process.memoryUsage().rss;
-        fn();
-        const endTime = performance.now();
-        const endMemory = process.memoryUsage().rss;
-        const duration = endTime - startTime;
-        const memory = endMemory - startMemory;
-
-        results.push({ duration, memory });
-    }, count);
-
-    const medianDuration = median(map(prop('duration'), results));
-    const medianMemory = median(map(prop('memory'), results));
-
-    return { medianDuration, medianMemory };
-}
+const { runBenchmark, cpuSpeed, clearRequireCache } = require('./measure');
 
 describe('startup / require time', () => {
     it('should not take longer as the defined budget to require the plugin', () => {

package.json

@@ -17,7 +17,7 @@
         "test": "npm run test:unit:with-coverage && npm run test:bench",
         "test:unit": "mocha test --recursive --reporter dot",
         "test:unit:with-coverage": "nyc npm run test:unit",
-        "test:bench": "mocha -t 10000 benchmarks",
+        "test:bench": "mocha -t 1200000 benchmarks",
         "coveralls": "cat ./build/coverage/lcov.info | coveralls",
         "changelog": "pr-log"
     },