diff --git a/.eslintrc.yml b/.eslintrc.yml index 9f8b14af02..e2229d6546 100644 --- a/.eslintrc.yml +++ b/.eslintrc.yml @@ -358,6 +358,7 @@ overrides: parserOptions: sourceType: script rules: + no-await-in-loop: off no-restricted-syntax: off no-console: off no-sync: off diff --git a/package.json b/package.json index d72058101b..b01d81d1cd 100644 --- a/package.json +++ b/package.json @@ -28,7 +28,7 @@ "testonly": "mocha --full-trace src/**/__tests__/**/*-test.js", "testonly:cover": "nyc npm run testonly", "lint": "eslint --cache --report-unused-disable-directives src resources", - "benchmark": "node ./resources/benchmark.js", + "benchmark": "node --predictable ./resources/benchmark.js", "prettier": "prettier --ignore-path .gitignore --write --list-different '**/*.{js,md,json,yml}'", "prettier:check": "prettier --ignore-path .gitignore --check '**/*.{js,md,json,yml}'", "check": "flow check", diff --git a/resources/benchmark-fork.js b/resources/benchmark-fork.js new file mode 100644 index 0000000000..ac78e9e1d2 --- /dev/null +++ b/resources/benchmark-fork.js @@ -0,0 +1,50 @@ +// @noflow + +'use strict'; + +const assert = require('assert'); +const cp = require('child_process'); + +// Clocks the time taken to execute a test per cycle (secs). +function clock(count, fn) { + const start = process.hrtime.bigint(); + for (let i = 0; i < count; ++i) { + fn(); + } + return Number(process.hrtime.bigint() - start) / count; +} + +if (require.main === module) { + const modulePath = process.env.BENCHMARK_MODULE_PATH; + assert(typeof modulePath === 'string'); + assert(process.send); + const module = require(modulePath); + + clock(7, module.measure); // warm up + process.nextTick(() => { + process.send({ + name: module.name, + clocked: clock(module.count, module.measure), + }); + }); +} + +function sampleModule(modulePath) { + return new Promise((resolve, reject) => { + const env = { BENCHMARK_MODULE_PATH: modulePath }; + const child = cp.fork(__filename, { env }); + let message; + let error; + + child.on('message', msg => (message = msg)); + child.on('error', e => (error = e)); + child.on('close', () => { + if (message) { + return resolve(message); + } + reject(error || new Error('Forked process closed without error')); + }); + }); +} + +module.exports = { sampleModule }; diff --git a/resources/benchmark.js b/resources/benchmark.js index 9810ffef70..244dacf50b 100644 --- a/resources/benchmark.js +++ b/resources/benchmark.js @@ -16,17 +16,15 @@ const { mkdirRecursive, readdirRecursive, } = require('./utils'); +const { sampleModule } = require('./benchmark-fork'); const NS_PER_SEC = 1e9; const LOCAL = 'local'; -const minTime = 0.05 * NS_PER_SEC; -// The maximum time a benchmark is allowed to run before finishing. -const maxTime = 5 * NS_PER_SEC; +// The maximum time in secounds a benchmark is allowed to run before finishing. +const maxTime = 5; // The minimum sample size required to perform statistical analysis. -const minSamples = 15; -// The default number of times to execute a test on a benchmark's first cycle. -const initCount = 10; +const minSamples = 5; function LOCAL_DIR(...paths) { return path.join(__dirname, '..', ...paths); @@ -97,44 +95,19 @@ function findFiles(cwd, pattern) { return out.split('\n').filter(Boolean); } -function collectSamples(fn) { - clock(initCount, fn); // initial warm up - - // Cycles a benchmark until a run `count` can be established. - // Resolve time span required to achieve a percent uncertainty of at most 1%. - // For more information see http://spiff.rit.edu/classes/phys273/uncert/uncert.html. - let count = initCount; - let clocked = 0; - while ((clocked = clock(count, fn)) < minTime) { - // Calculate how many more iterations it will take to achieve the `minTime`. - count += Math.ceil(((minTime - clocked) * count) / clocked); - } - - let elapsed = 0; +async function collectSamples(modulePath) { const samples = []; // If time permits, increase sample size to reduce the margin of error. - while (samples.length < minSamples || elapsed < maxTime) { - clocked = clock(count, fn); + const start = Date.now(); + while (samples.length < minSamples || (Date.now() - start) / 1e3 < maxTime) { + const { clocked } = await sampleModule(modulePath); assert(clocked > 0); - - elapsed += clocked; - // Compute the seconds per operation. - samples.push(clocked / count); + samples.push(clocked); } - return samples; } -// Clocks the time taken to execute a test per cycle (secs). -function clock(count, fn) { - const start = process.hrtime.bigint(); - for (let i = 0; i < count; ++i) { - fn(); - } - return Number(process.hrtime.bigint() - start); -} - // T-Distribution two-tailed critical values for 95% confidence. // See http://www.itl.nist.gov/div898/handbook/eda/section3/eda3672.htm. const tTable = /* prettier-ignore */ { @@ -238,7 +211,7 @@ function maxBy(array, fn) { } // Prepare all revisions and run benchmarks matching a pattern against them. -function prepareAndRunBenchmarks(benchmarkPatterns, revisions) { +async function prepareAndRunBenchmarks(benchmarkPatterns, revisions) { const environments = revisions.map(revision => ({ revision, distPath: prepareRevision(revision), @@ -248,14 +221,16 @@ function prepareAndRunBenchmarks(benchmarkPatterns, revisions) { const results = []; for (let i = 0; i < environments.length; ++i) { const environment = environments[i]; - const module = require(path.join(environment.distPath, benchmark)); + const modulePath = path.join(environment.distPath, benchmark); - if (i) { - console.log('⏱️ ' + module.name); + if (i === 0) { + const { name } = await sampleModule(modulePath); + console.log('⏱️ ' + name); } try { - const samples = collectSamples(module.measure); + const samples = await collectSamples(modulePath); + results.push({ name: environment.revision, samples, @@ -263,7 +238,7 @@ function prepareAndRunBenchmarks(benchmarkPatterns, revisions) { }); process.stdout.write(' ' + cyan(i + 1) + ' tests completed.\u000D'); } catch (error) { - console.log(' ' + module.name + ': ' + red(String(error))); + console.log(' ' + environment.revision + ': ' + red(String(error))); } } console.log('\n'); diff --git a/src/language/__tests__/parser-benchmark.js b/src/language/__tests__/parser-benchmark.js index 9f301af59e..36cf68582c 100644 --- a/src/language/__tests__/parser-benchmark.js +++ b/src/language/__tests__/parser-benchmark.js @@ -4,6 +4,7 @@ import { kitchenSinkQuery } from '../../__fixtures__'; import { parse } from '../parser'; export const name = 'Parse kitchen sink'; +export const count = 1000; export function measure() { parse(kitchenSinkQuery); } diff --git a/src/utilities/__tests__/buildASTSchema-benchmark.js b/src/utilities/__tests__/buildASTSchema-benchmark.js index 31beb30d12..5ccd40e238 100644 --- a/src/utilities/__tests__/buildASTSchema-benchmark.js +++ b/src/utilities/__tests__/buildASTSchema-benchmark.js @@ -8,6 +8,7 @@ import { buildASTSchema } from '../buildASTSchema'; const schemaAST = parse(bigSchemaSDL); export const name = 'Build Schema from AST'; +export const count = 10; export function measure() { buildASTSchema(schemaAST, { assumeValid: true }); } diff --git a/src/utilities/__tests__/buildClientSchema-benchmark.js b/src/utilities/__tests__/buildClientSchema-benchmark.js index c566f563c9..39872f4c5c 100644 --- a/src/utilities/__tests__/buildClientSchema-benchmark.js +++ b/src/utilities/__tests__/buildClientSchema-benchmark.js @@ -5,6 +5,7 @@ import { bigSchemaIntrospectionResult } from '../../__fixtures__'; import { buildClientSchema } from '../buildClientSchema'; export const name = 'Build Schema from Introspection'; +export const count = 10; export function measure() { buildClientSchema(bigSchemaIntrospectionResult.data, { assumeValid: true }); } diff --git a/src/utilities/__tests__/introspectionFromSchema-benchmark.js b/src/utilities/__tests__/introspectionFromSchema-benchmark.js index 4d1b79bbcf..3f681019cd 100644 --- a/src/utilities/__tests__/introspectionFromSchema-benchmark.js +++ b/src/utilities/__tests__/introspectionFromSchema-benchmark.js @@ -10,6 +10,7 @@ const queryAST = parse(getIntrospectionQuery()); const schema = buildSchema(bigSchemaSDL, { assumeValid: true }); export const name = 'Execute Introspection Query'; +export const count = 10; export function measure() { execute(schema, queryAST); } diff --git a/src/validation/__tests__/validateGQL-benchmark.js b/src/validation/__tests__/validateGQL-benchmark.js index f12d7d5f28..90f6271d25 100644 --- a/src/validation/__tests__/validateGQL-benchmark.js +++ b/src/validation/__tests__/validateGQL-benchmark.js @@ -9,6 +9,7 @@ const schema = buildSchema(bigSchemaSDL, { assumeValid: true }); const queryAST = parse(getIntrospectionQuery()); export const name = 'Validate Introspection Query'; +export const count = 50; export function measure() { validate(schema, queryAST); } diff --git a/src/validation/__tests__/validateSDL-benchmark.js b/src/validation/__tests__/validateSDL-benchmark.js index 8d1eaadbdc..7087d832d8 100644 --- a/src/validation/__tests__/validateSDL-benchmark.js +++ b/src/validation/__tests__/validateSDL-benchmark.js @@ -8,6 +8,7 @@ import { validateSDL } from '../validate'; const sdlAST = parse(bigSchemaSDL); export const name = 'Validate SDL Document'; +export const count = 10; export function measure() { validateSDL(sdlAST); }