diff --git a/lib/runner.js b/lib/runner.js index 37c7211ab..fe240cb26 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -31,6 +31,7 @@ class Runner extends Emittery { this.boundCompareTestSnapshot = this.compareTestSnapshot.bind(this); this.interrupted = false; this.snapshots = null; + this.nextTaskIndex = 0; this.tasks = { after: [], afterAlways: [], @@ -76,6 +77,8 @@ class Runner extends Emittery { }); } + metadata.taskIndex = this.nextTaskIndex++; + const {args, buildTitle, implementations, rawTitle} = parseTestArgs(testArgs); if (this.checkSelectedByLineNumbers) { @@ -285,7 +288,7 @@ class Runner extends Emittery { return result; } - async runHooks(tasks, contextRef, titleSuffix, testPassed) { + async runHooks(tasks, contextRef, {titleSuffix, testPassed, associatedTaskIndex} = {}) { const hooks = tasks.map(task => new Runnable({ contextRef, experiments: this.experiments, @@ -295,7 +298,7 @@ class Runner extends Emittery { t => task.implementation.apply(null, [t].concat(task.args)), compareTestSnapshot: this.boundCompareTestSnapshot, updateSnapshots: this.updateSnapshots, - metadata: task.metadata, + metadata: {...task.metadata, associatedTaskIndex}, powerAssert: this.powerAssert, title: `${task.title}${titleSuffix || ''}`, isHook: true, @@ -326,7 +329,14 @@ class Runner extends Emittery { async runTest(task, contextRef) { const hookSuffix = ` for ${task.title}`; - let hooksOk = await this.runHooks(this.tasks.beforeEach, contextRef, hookSuffix); + let hooksOk = await this.runHooks( + this.tasks.beforeEach, + contextRef, + { + titleSuffix: hookSuffix, + associatedTaskIndex: task.metadata.taskIndex + } + ); let testOk = false; if (hooksOk) { @@ -358,7 +368,14 @@ class Runner extends Emittery { logs: result.logs }); - hooksOk = await this.runHooks(this.tasks.afterEach, contextRef, hookSuffix, testOk); + hooksOk = await this.runHooks( + this.tasks.afterEach, + contextRef, + { + titleSuffix: hookSuffix, + testPassed: testOk, + associatedTaskIndex: task.metadata.taskIndex + }); } else { this.emit('stateChange', { type: 'test-failed', @@ -372,7 +389,14 @@ class Runner extends Emittery { } } - const alwaysOk = await this.runHooks(this.tasks.afterEachAlways, contextRef, hookSuffix, testOk); + const alwaysOk = await this.runHooks( + this.tasks.afterEachAlways, + contextRef, + { + titleSuffix: hookSuffix, + testPassed: testOk, + associatedTaskIndex: task.metadata.taskIndex + }); return alwaysOk && hooksOk && testOk; } diff --git a/lib/snapshot-manager.js b/lib/snapshot-manager.js index 1871977c9..119476452 100644 --- a/lib/snapshot-manager.js +++ b/lib/snapshot-manager.js @@ -104,13 +104,32 @@ function combineEntries(entries) { const buffers = []; let byteLength = 0; - const sortedKeys = [...entries.keys()].sort(); + const sortedKeys = [...entries.keys()].sort((keyA, keyB) => { + const [a, b] = [entries.get(keyA), entries.get(keyB)]; + const taskDifference = a.taskIndex - b.taskIndex; + + if (taskDifference !== 0) { + return taskDifference; + } + + const [assocA, assocB] = [a.associatedTaskIndex, b.associatedTaskIndex]; + if (assocA !== undefined && assocB !== undefined) { + const assocDifference = assocA - assocB; + + if (assocDifference !== 0) { + return assocDifference; + } + } + + return a.snapIndex - b.snapIndex; + }); + for (const key of sortedKeys) { const keyBuffer = Buffer.from(`\n\n## ${key}\n\n`, 'utf8'); buffers.push(keyBuffer); byteLength += keyBuffer.byteLength; - const formattedEntries = entries.get(key); + const formattedEntries = entries.get(key).buffers; const last = formattedEntries[formattedEntries.length - 1]; for (const entry of formattedEntries) { buffers.push(entry); @@ -176,10 +195,11 @@ function encodeSnapshots(buffersByHash) { byteOffset += 2; const entries = []; - for (const pair of buffersByHash) { - const hash = pair[0]; - const snapshotBuffers = pair[1]; - + // Maps can't have duplicate keys, so all items in [...buffersByHash.keys()] + // are unique, so sortedHashes should be deterministic. + const sortedHashes = [...buffersByHash.keys()].sort(); + const sortedBuffersByHash = [...sortedHashes.map(hash => [hash, buffersByHash.get(hash)])]; + for (const [hash, snapshotBuffers] of sortedBuffersByHash) { buffers.push(Buffer.from(hash, 'hex')); byteOffset += MD5_HASH_LENGTH; @@ -332,6 +352,7 @@ class Manager { const descriptor = concordance.describe(options.expected, concordanceOptions); const snapshot = concordance.serialize(descriptor); const entry = formatEntry(options.label, descriptor); + const {taskIndex, snapIndex, associatedTaskIndex} = options; return () => { // Must be called in order! this.hasChanges = true; @@ -353,9 +374,9 @@ class Manager { snapshots.push(snapshot); if (this.reportEntries.has(options.belongsTo)) { - this.reportEntries.get(options.belongsTo).push(entry); + this.reportEntries.get(options.belongsTo).buffers.push(entry); } else { - this.reportEntries.set(options.belongsTo, [entry]); + this.reportEntries.set(options.belongsTo, {buffers: [entry], taskIndex, snapIndex, associatedTaskIndex}); } }; } diff --git a/lib/test.js b/lib/test.js index e8058b115..b0814654e 100644 --- a/lib/test.js +++ b/lib/test.js @@ -230,7 +230,17 @@ class Test { const index = id ? 0 : this.nextSnapshotIndex++; const label = id ? '' : message || `Snapshot ${index + 1}`; // Human-readable labels start counting at 1. - const {record, ...result} = options.compareTestSnapshot({belongsTo, deferRecording, expected, index, label}); + const {taskIndex, associatedTaskIndex} = this.metadata; + const {record, ...result} = options.compareTestSnapshot({ + belongsTo, + deferRecording, + expected, + index, + label, + taskIndex, + snapIndex: this.snapshotCount, + associatedTaskIndex + }); if (record) { this.deferredSnapshotRecordings.push(record); } diff --git a/test/snapshot-order/fixtures/intertest-order/package.json b/test/snapshot-order/fixtures/intertest-order/package.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/test/snapshot-order/fixtures/intertest-order/package.json @@ -0,0 +1 @@ +{} diff --git a/test/snapshot-order/fixtures/intertest-order/test.js b/test/snapshot-order/fixtures/intertest-order/test.js new file mode 100644 index 000000000..79081b23b --- /dev/null +++ b/test/snapshot-order/fixtures/intertest-order/test.js @@ -0,0 +1,35 @@ +const test = require('ava'); + +const reverse = process.env.INTERTEST_ORDER_REVERSE; + +// Functions which resolve the corresponding promise to undefined +let resolveOne; +let resolveTwo; + +// Promises with triggerable resolution +const one = new Promise(resolve => { + resolveOne = resolve; +}); + +const two = new Promise(resolve => { + resolveTwo = resolve; +}); + +// Test cases which await the triggerable promises +test('one', async t => { + await one; + t.snapshot('one'); + resolveTwo(); +}); +test('two', async t => { + await two; + t.snapshot('two'); + resolveOne(); +}); + +// Start resolution +if (reverse) { + resolveTwo(); +} else { + resolveOne(); +} diff --git a/test/snapshot-order/fixtures/randomness/package.json b/test/snapshot-order/fixtures/randomness/package.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/test/snapshot-order/fixtures/randomness/package.json @@ -0,0 +1 @@ +{} diff --git a/test/snapshot-order/fixtures/randomness/test.js b/test/snapshot-order/fixtures/randomness/test.js new file mode 100644 index 000000000..d3d1fa242 --- /dev/null +++ b/test/snapshot-order/fixtures/randomness/test.js @@ -0,0 +1,67 @@ +const test = require('ava'); + +const id = index => `index: ${index}`; +const randomDelay = () => new Promise(resolve => { + setTimeout(resolve, Math.random() * 1000); +}); + +test.before(async t => { + await randomDelay(); + t.snapshot(id(-2), 'in a before hook'); +}); + +test.beforeEach(async t => { + await randomDelay(); + t.snapshot(id(-1.5), 'in a beforeEach hook'); +}); + +test.afterEach(async t => { + await randomDelay(); + t.snapshot(id(-1), 'in an afterEach hook'); +}); + +test.afterEach.always(async t => { + await randomDelay(); + t.snapshot(id(-0.5), 'in an afterEachAlways hook'); +}); + +test('B - declare some snapshots', async t => { + await randomDelay(); + t.snapshot(id(0)); + t.snapshot(id(1), 'has a message'); + t.snapshot(id(2), 'also has a message'); + t.snapshot(id(3), {id: 'has an ID'}); +}); + +test('A - declare some more snapshots', async t => { + await randomDelay(); + t.snapshot(id(4)); +}); + +test('C - declare some snapshots in a try()', async t => { + await randomDelay(); + t.snapshot(id(5), 'outer'); + (await t.try('trying', t => { + t.snapshot(id(6), 'inner'); + })).commit(); + t.snapshot(id(7), 'outer again'); +}); + +test('E - discard some snapshots in a try()', async t => { + await randomDelay(); + t.snapshot(id(8), 'outer'); + (await t.try('trying', t => { + t.snapshot(id(9), 'inner'); + })).discard(); + t.snapshot(id(10), 'outer again'); +}); + +test('D - more snapshots with IDs', async t => { + await randomDelay(); + t.snapshot(id(11), {id: 'the first in test D'}); + t.snapshot(id(12)); + // These have to be reported in reverse declaration order, because they can't + // be reported under the same header + t.snapshot(id(14), {id: 'the second-to-last in test D'}); + t.snapshot(id(13)); +}); diff --git a/test/snapshot-order/fixtures/randomness/test.js.md b/test/snapshot-order/fixtures/randomness/test.js.md new file mode 100644 index 000000000..14c2251f4 --- /dev/null +++ b/test/snapshot-order/fixtures/randomness/test.js.md @@ -0,0 +1,167 @@ +# Snapshot report for `test.js` + +The actual snapshot is saved in `test.js.snap`. + +Generated by [AVA](https://avajs.dev). + +## before hook + +> in a before hook + + 'index: -2' + +## beforeEach hook for B - declare some snapshots + +> in a beforeEach hook + + 'index: -1.5' + +## beforeEach hook for A - declare some more snapshots + +> in a beforeEach hook + + 'index: -1.5' + +## beforeEach hook for C - declare some snapshots in a try() + +> in a beforeEach hook + + 'index: -1.5' + +## beforeEach hook for E - discard some snapshots in a try() + +> in a beforeEach hook + + 'index: -1.5' + +## beforeEach hook for D - more snapshots with IDs + +> in a beforeEach hook + + 'index: -1.5' + +## afterEach hook for B - declare some snapshots + +> in an afterEach hook + + 'index: -1' + +## afterEach hook for A - declare some more snapshots + +> in an afterEach hook + + 'index: -1' + +## afterEach hook for C - declare some snapshots in a try() + +> in an afterEach hook + + 'index: -1' + +## afterEach hook for E - discard some snapshots in a try() + +> in an afterEach hook + + 'index: -1' + +## afterEach hook for D - more snapshots with IDs + +> in an afterEach hook + + 'index: -1' + +## afterEach.always hook for B - declare some snapshots + +> in an afterEachAlways hook + + 'index: -0.5' + +## afterEach.always hook for A - declare some more snapshots + +> in an afterEachAlways hook + + 'index: -0.5' + +## afterEach.always hook for C - declare some snapshots in a try() + +> in an afterEachAlways hook + + 'index: -0.5' + +## afterEach.always hook for E - discard some snapshots in a try() + +> in an afterEachAlways hook + + 'index: -0.5' + +## afterEach.always hook for D - more snapshots with IDs + +> in an afterEachAlways hook + + 'index: -0.5' + +## B - declare some snapshots + +> Snapshot 1 + + 'index: 0' + +> has a message + + 'index: 1' + +> also has a message + + 'index: 2' + +## has an ID + + 'index: 3' + +## A - declare some more snapshots + +> Snapshot 1 + + 'index: 4' + +## C - declare some snapshots in a try() + +> outer + + 'index: 5' + +> inner + + 'index: 6' + +> outer again + + 'index: 7' + +## E - discard some snapshots in a try() + +> outer + + 'index: 8' + +> outer again + + 'index: 10' + +## the first in test D + + 'index: 11' + +## D - more snapshots with IDs + +> Snapshot 1 + + 'index: 12' + +> Snapshot 2 + + 'index: 13' + +## the second-to-last in test D + + 'index: 14' diff --git a/test/snapshot-order/fixtures/randomness/test.js.snap b/test/snapshot-order/fixtures/randomness/test.js.snap new file mode 100644 index 000000000..3082a112d Binary files /dev/null and b/test/snapshot-order/fixtures/randomness/test.js.snap differ diff --git a/test/snapshot-order/fixtures/report-declaration-order/package.json b/test/snapshot-order/fixtures/report-declaration-order/package.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/test/snapshot-order/fixtures/report-declaration-order/package.json @@ -0,0 +1 @@ +{} diff --git a/test/snapshot-order/fixtures/report-declaration-order/test.js b/test/snapshot-order/fixtures/report-declaration-order/test.js new file mode 100644 index 000000000..b25902d90 --- /dev/null +++ b/test/snapshot-order/fixtures/report-declaration-order/test.js @@ -0,0 +1,55 @@ +const test = require('ava'); + +const id = index => `index: ${index}`; + +test.before(t => { + t.snapshot(id(-2), 'in a before hook'); +}); + +test.beforeEach(t => { + t.snapshot(id(-1.5), 'in a beforeEach hook'); +}); + +test.afterEach(t => { + t.snapshot(id(-1), 'in an afterEach hook'); +}); + +test.afterEach.always(t => { + t.snapshot(id(-0.5), 'in an afterEachAlways hook'); +}); + +test('B - declare some snapshots', t => { + t.snapshot(id(0)); + t.snapshot(id(1), 'has a message'); + t.snapshot(id(2), 'also has a message'); + t.snapshot(id(3), {id: 'has an ID'}); +}); + +test('A - declare some more snapshots', t => { + t.snapshot(id(4)); +}); + +test('C - declare some snapshots in a try()', async t => { + t.snapshot(id(5), 'outer'); + (await t.try('trying', t => { + t.snapshot(id(6), 'inner'); + })).commit(); + t.snapshot(id(7), 'outer again'); +}); + +test('E - discard some snapshots in a try()', async t => { + t.snapshot(id(8), 'outer'); + (await t.try('trying', t => { + t.snapshot(id(9), 'inner'); + })).discard(); + t.snapshot(id(10), 'outer again'); +}); + +test('D - more snapshots with IDs', t => { + t.snapshot(id(11), {id: 'the first in test D'}); + t.snapshot(id(12)); + // These have to be reported in reverse declaration order, because they can't + // be reported under the same header + t.snapshot(id(14), {id: 'the second-to-last in test D'}); + t.snapshot(id(13)); +}); diff --git a/test/snapshot-order/helpers/get-snapshot-ids.js b/test/snapshot-order/helpers/get-snapshot-ids.js new file mode 100644 index 000000000..00c376b4c --- /dev/null +++ b/test/snapshot-order/helpers/get-snapshot-ids.js @@ -0,0 +1,14 @@ +function getSnapshotIds(report) { + function * matchAll(string, regexp) { + let match; + while ((match = regexp.exec(string)) !== null) { + yield match; + } + } + + const ids = [...matchAll(report, /'index: ([-.\d]+)'/g)].map(match => Number(match[1])); + + return ids; +} + +module.exports = getSnapshotIds; diff --git a/test/snapshot-order/intertest-order.js b/test/snapshot-order/intertest-order.js new file mode 100644 index 000000000..6324f92b7 --- /dev/null +++ b/test/snapshot-order/intertest-order.js @@ -0,0 +1,42 @@ +const test = require('@ava/test'); +const exec = require('../helpers/exec'); +const fs = require('fs'); +const path = require('path'); + +test('snapshot files are independent of test resolution order', async t => { + const options = { + cwd: exec.cwd('intertest-order'), + env: { + AVA_FORCE_CI: 'not-ci' + } + }; + + const snapshotPath = path.join(options.cwd, 'test.js.snap'); + + // Schedule snapshot cleanup + t.teardown(() => { + fs.unlinkSync(snapshotPath); + fs.unlinkSync(path.join(options.cwd, 'test.js.md')); + }); + + // Run, updating snapshots. + await exec.fixture(['test.js', '--update-snapshots'], options); + + // Read the resulting file + const snapshot = fs.readFileSync(snapshotPath); + + // Run in reversed order, updating snapshots. + await exec.fixture(['test.js', '--update-snapshots'], { + ...options, + env: { + INTERTEST_ORDER_REVERSE: 'true', + ...options.env + } + }); + + // Read the resulting file + const snapshotReversed = fs.readFileSync(snapshotPath); + + // Compare snapshots + t.deepEqual(snapshot, snapshotReversed); +}); diff --git a/test/snapshot-order/randomness.js b/test/snapshot-order/randomness.js new file mode 100644 index 000000000..92078f907 --- /dev/null +++ b/test/snapshot-order/randomness.js @@ -0,0 +1,31 @@ +const test = require('@ava/test'); +const exec = require('../helpers/exec'); +const fs = require('fs'); +const path = require('path'); +const getSnapshotIds = require('./helpers/get-snapshot-ids'); + +test('deterministic and sorted over a large, random test case', async t => { + const options = { + cwd: exec.cwd('randomness'), + env: { + AVA_FORCE_CI: 'not-ci' + } + }; + + const snapshotPath = path.join(options.cwd, 'test.js.snap'); + const reportPath = path.join(options.cwd, 'test.js.md'); + + // Run test + await exec.fixture(['--update-snapshots'], options); + + // Assert snapshot is unchanged + const snapshot = fs.readFileSync(snapshotPath); + + t.snapshot(snapshot, 'resulting snapshot in binary encoding'); + + // Assert report is sorted + const report = fs.readFileSync(reportPath); + const ids = getSnapshotIds(report); + + t.deepEqual(ids, [...ids].sort((a, b) => a - b)); +}); diff --git a/test/snapshot-order/report-declaration-order.js b/test/snapshot-order/report-declaration-order.js new file mode 100644 index 000000000..62eb85e44 --- /dev/null +++ b/test/snapshot-order/report-declaration-order.js @@ -0,0 +1,29 @@ +const test = require('@ava/test'); +const exec = require('../helpers/exec'); +const fs = require('fs'); +const path = require('path'); +const getSnapshotIds = require('./helpers/get-snapshot-ids'); + +test('snapshot reports are sorted in declaration order', async t => { + const options = { + cwd: exec.cwd('report-declaration-order'), + env: { + AVA_FORCE_CI: 'not-ci' + } + }; + + // Scehdule snapshot cleanup + t.teardown(() => { + fs.unlinkSync(path.join(options.cwd, 'test.js.snap')); + fs.unlinkSync(reportPath); + }); + + await exec.fixture(['--update-snapshots'], options); + + const reportPath = path.join(options.cwd, 'test.js.md'); + + const report = fs.readFileSync(reportPath, {encoding: 'utf8'}); + const ids = getSnapshotIds(report); + + t.deepEqual(ids, [...ids].sort((a, b) => a - b)); +}); diff --git a/test/snapshot-order/snapshots/randomness.js.md b/test/snapshot-order/snapshots/randomness.js.md new file mode 100644 index 000000000..14016b0e8 --- /dev/null +++ b/test/snapshot-order/snapshots/randomness.js.md @@ -0,0 +1,35 @@ +# Snapshot report for `test/snapshot-order/randomness.js` + +The actual snapshot is saved in `randomness.js.snap`. + +Generated by [AVA](https://avajs.dev). + +## deterministic and sorted over a large, random test case + +> Snapshot 1 + + Buffer @Uint8Array [ + 41564120 536e6170 73686f74 2076320a 020019c4 186451f5 248fc7d2 9bc1c0e4 + 9b6a1f8b 08000000 00000003 5bc6c4c0 20c1c0f0 66f53aff 9b6187ad 7ef5c6ab + 69395f66 64000171 2096889b cea1f1e9 cc6abe6a 8e3f7d1f 986418c1 a27a402c + f92db345 dc7c62a0 c041a66b d91bd7ef 64048bba 02b1aac3 da1d160f fc2bce6e + 2c0e6758 7caf8611 2c1a0792 79f85f47 95e7679d 64a4c26c 05cd09a5 8c60d152 + 20d660e4 5eb73253 60dffb96 e3159d5a 47bf3182 45fb80d8 dae87dec d21fce25 + a16953cf ed927c7c 9f112cba 1c886d8a c3372ce4 345bbcf8 7ee9e682 a52fa731 + 82450f80 640e7f35 58673293 ad63ff91 8f6c1d69 b719c1a2 3781384c 37e7c7ff + f90717b9 44cd5ee3 cd709e8b 112cfa09 8823fc12 73fd7cad 27c83e5c ea6e79d9 + df9b112c ca0d0c8a d8d04fa2 b5e55b9e dd7f7a6d 5b7b9946 22235854 1188e3d7 + de5d29f8 9edd4fae fbdaa4dc 33acff98 c1a2e650 ec0bc5c9 409ce62a 6725c693 + a66d716f fe29dbe3 0b0e3282 458141c3 90f9a478 bf83ee83 7ba6775e ad991632 + e5192358 742a1097 ce5b9b6a c12dc2a7 cef5532a 8525f52a 23587435 10371dd0 + 7efe5874 b1e8c4fd cc9167de cfdcc508 163d04c4 93ac7e68 78dc5d7c c794a9d1 + f95ba6de 0346b0e8 4d209ebc ffafc6d4 57fce1e5 e1378236 6d5c15c8 0416fd00 + c5ecc074 70c995f9 e6ee835b 535b27ae a8dad95b a4cb0416 95856213 908a33cf + 6d74158e 32ae2ae8 cc37f936 7d292358 d41b885f da3eac39 e977af69 a6fda7b0 + e70a8c9e 8c60d114 20fe2230 5fdde7e0 7cfb673e 8fc5773e d0ef6006 8b5641f1 + 04285e06 c4bfbeda 79ee3a24 7cb8f998 d876d6f3 071918c1 a27b8198 99811d9c + 22195919 04193933 f352522b ac14740d 49113634 4111e686 a936d033 c56e8c11 + 71eae112 86b82448 d741b404 0754c204 ab2876b5 665845cd 29743bcc 1c63ec81 + 4f5a5419 611736c6 6aa30576 c506c4d8 88d39f30 c30db08a 1a6215c5 ee6a5d43 + 00e09b96 2e630500 00 + ] diff --git a/test/snapshot-order/snapshots/randomness.js.snap b/test/snapshot-order/snapshots/randomness.js.snap new file mode 100644 index 000000000..5b29a3ebd Binary files /dev/null and b/test/snapshot-order/snapshots/randomness.js.snap differ