From 0804107b49ef3bb43656cd48d27b0d54ea080d71 Mon Sep 17 00:00:00 2001 From: Mark Wubben Date: Mon, 22 Apr 2019 12:37:30 +0200 Subject: [PATCH] Fail snapshot assertions in CI if snapshot is missing Fixes #1585. --- lib/api.js | 4 +++- lib/assert.js | 3 ++- lib/runner.js | 2 ++ lib/snapshot-manager.js | 8 ++++++- lib/worker/subprocess.js | 1 + test/integration/snapshots.js | 43 +++++++++++++++++++++++++++++++---- 6 files changed, 53 insertions(+), 8 deletions(-) diff --git a/lib/api.js b/lib/api.js index 52095b81b..9ee9f1d32 100644 --- a/lib/api.js +++ b/lib/api.js @@ -207,7 +207,9 @@ class Api extends Emittery { const execArgv = await this._computeForkExecArgv(); const options = { - ...apiOptions, // If we're looking for matches, run every single test process in exclusive-only mode + ...apiOptions, + recordNewSnapshots: !isCi, + // If we're looking for matches, run every single test process in exclusive-only mode runOnlyExclusive: apiOptions.match.length > 0 || runtimeOptions.runOnlyExclusive === true }; if (precompilation) { diff --git a/lib/assert.js b/lib/assert.js index 33dd64693..a7cfd5701 100644 --- a/lib/assert.js +++ b/lib/assert.js @@ -616,9 +616,10 @@ class Assertions { values: [formatDescriptorDiff(result.actual, result.expected, {invert: true})] })); } else { + // This can only occur in CI environments. fail(new AssertionError({ assertion: 'snapshot', - message: message || 'No snapshot available, run with --update-snapshots' + message: message || 'No snapshot available — new snapshots are not created in CI environments' })); } }); diff --git a/lib/runner.js b/lib/runner.js index 6457fbaff..18af02f7a 100644 --- a/lib/runner.js +++ b/lib/runner.js @@ -16,6 +16,7 @@ class Runner extends Emittery { this.file = options.file; this.match = options.match || []; this.projectDir = options.projectDir; + this.recordNewSnapshots = options.recordNewSnapshots === true; this.runOnlyExclusive = options.runOnlyExclusive === true; this.serial = options.serial === true; this.snapshotDir = options.snapshotDir; @@ -171,6 +172,7 @@ class Runner extends Emittery { file: this.file, fixedLocation: this.snapshotDir, projectDir: this.projectDir, + recordNewSnapshots: this.recordNewSnapshots, updating: this.updateSnapshots }); this.emit('dependency', this.snapshots.snapPath); diff --git a/lib/snapshot-manager.js b/lib/snapshot-manager.js index 800daabd8..bd0201fc6 100644 --- a/lib/snapshot-manager.js +++ b/lib/snapshot-manager.js @@ -291,6 +291,7 @@ class Manager { constructor(options) { this.appendOnly = options.appendOnly; this.dir = options.dir; + this.recordNewSnapshots = options.recordNewSnapshots; this.relFile = options.relFile; this.reportFile = options.reportFile; this.snapFile = options.snapFile; @@ -309,6 +310,10 @@ class Manager { } if (options.index === entries.length) { + if (!this.recordNewSnapshots) { + return {pass: false}; + } + this.record(hash, options); return {pass: true}; } @@ -400,7 +405,7 @@ function resolveSourceFile(file) { return file; } -function load({file, fixedLocation, projectDir, updating}) { +function load({file, fixedLocation, projectDir, recordNewSnapshots, updating}) { const sourceFile = resolveSourceFile(file); const dir = determineSnapshotDir({file: sourceFile, fixedLocation, projectDir}); const relFile = path.relative(projectDir, sourceFile); @@ -424,6 +429,7 @@ function load({file, fixedLocation, projectDir, updating}) { return new Manager({ appendOnly, dir, + recordNewSnapshots, relFile, reportFile, snapFile, diff --git a/lib/worker/subprocess.js b/lib/worker/subprocess.js index 4b0497ff4..4a31fdb32 100644 --- a/lib/worker/subprocess.js +++ b/lib/worker/subprocess.js @@ -36,6 +36,7 @@ ipc.options.then(options => { file: options.file, match: options.match, projectDir: options.projectDir, + recordNewSnapshots: options.recordNewSnapshots, runOnlyExclusive: options.runOnlyExclusive, serial: options.serial, snapshotDir: options.snapshotDir, diff --git a/test/integration/snapshots.js b/test/integration/snapshots.js index f9f7a62cc..d1ffe2144 100644 --- a/test/integration/snapshots.js +++ b/test/integration/snapshots.js @@ -6,6 +6,12 @@ const uniqueTempDir = require('unique-temp-dir'); const {test} = require('tap'); const {execCli} = require('../helper/cli'); +const overrideCIChecks = { + CI: '', + TRAVIS: '', + CONTINUOUS_INTEGRATION: '' +}; + for (const obj of [ {type: 'colocated', rel: '', dir: ''}, {type: '__tests__', rel: '__tests__-dir', dir: '__tests__/__snapshots__'}, @@ -24,7 +30,7 @@ for (const obj of [ const dirname = path.join('fixture/snapshots', obj.rel); // Test should pass, and a snapshot gets written - execCli(['--update-snapshots'], {dirname}, error => { + execCli(['--update-snapshots', '--verbose'], {dirname, env: overrideCIChecks}, error => { t.ifError(error); t.true(fs.existsSync(snapPath)); @@ -50,7 +56,7 @@ test('one', t => { })`; fs.writeFileSync(path.join(cwd, 'test.js'), initial); - const run = () => execa(process.execPath, [cliPath, '--verbose', '--no-color'], {cwd, env: {CI: '1'}, reject: false}); + const run = () => execa(process.execPath, [cliPath, '--verbose', '--no-color'], {cwd, env: overrideCIChecks, reject: false}); return run().then(result => { t.match(result.stdout, /1 test passed/); @@ -161,7 +167,7 @@ test('snapshots infer their location and name from sourcemaps', t => { t.true(fs.existsSync(relFilePath)); }; - execCli([], {dirname: relativeFixtureDir}, (error, stdout) => { + execCli(['--verbose'], {dirname: relativeFixtureDir, env: overrideCIChecks}, (error, stdout) => { t.ifError(error); snapFixtureFilePaths.forEach(x => verifySnapFixtureFiles(x)); t.match(stdout, /6 tests passed/); @@ -202,7 +208,7 @@ test('snapshots resolved location from "snapshotDir" in AVA config', t => { t.true(fs.existsSync(relFilePath)); }; - execCli([], {dirname: relativeFixtureDir}, (error, stdout) => { + execCli(['--verbose'], {dirname: relativeFixtureDir, env: overrideCIChecks}, (error, stdout) => { t.ifError(error); snapFixtureFilePaths.forEach(x => verifySnapFixtureFiles(x)); t.match(stdout, /6 tests passed/); @@ -231,7 +237,7 @@ test('snapshots are indentical on different platforms', t => { [reportPath, snapPath].forEach(fp => removeFile(fp)); // Test should pass, and a snapshot gets written - execCli(['--update-snapshots'], {dirname: fixtureDir}, error => { + execCli(['--update-snapshots', '--verbose'], {dirname: fixtureDir, env: overrideCIChecks}, error => { t.ifError(error); t.true(fs.existsSync(reportPath)); t.true(fs.existsSync(snapPath)); @@ -246,3 +252,30 @@ test('snapshots are indentical on different platforms', t => { t.end(); }); }); + +test('in CI, new snapshots are not recorded', t => { + const fixtureDir = path.join(__dirname, '..', 'fixture', 'snapshots', 'test-content'); + const reportPath = path.join(fixtureDir, 'tests', 'snapshots', 'test.js.md'); + const snapPath = path.join(fixtureDir, 'tests', 'snapshots', 'test.js.snap'); + + const removeFile = filePath => { + try { + fs.unlinkSync(filePath); + } catch (error) { + if (error.code !== 'ENOENT') { + throw error; + } + } + }; + + // Clear current snapshots + [reportPath, snapPath].forEach(fp => removeFile(fp)); + + // Test should fail, no snapshot gets written + execCli([], {dirname: fixtureDir}, (_, stdout) => { + t.match(stdout, 'No snapshot available — new snapshots are not created in CI environments'); + t.false(fs.existsSync(reportPath)); + t.false(fs.existsSync(snapPath)); + t.end(); + }); +});