Store screenshots of CI e2e failures as CI artifacts

.github/workflows/end2end-test.yml

@@ -51,6 +51,13 @@ jobs:
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 0' < ~/.jest-e2e-tests )
 
+    - name: Archive debug artifacts (screenshots, HTML snapshots)
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: failures-artifacts
+        path: artifacts
+
   admin-2:
     name: Admin - 2
 
@@ -92,6 +99,13 @@ jobs:
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 1' < ~/.jest-e2e-tests )
 
+    - name: Archive debug artifacts (screenshots, HTML snapshots)
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: failures-artifacts
+        path: artifacts
+
   admin-3:
     name: Admin - 3
 
@@ -133,6 +147,13 @@ jobs:
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 2' < ~/.jest-e2e-tests )
 
+    - name: Archive debug artifacts (screenshots, HTML snapshots)
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: failures-artifacts
+        path: artifacts
+
   admin-4:
     name: Admin - 4
 
@@ -173,3 +194,11 @@ jobs:
       run: |
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
         $( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 3' < ~/.jest-e2e-tests )
+
+    - name: Archive debug artifacts (screenshots, HTML snapshots)
+      uses: actions/upload-artifact@v2
+      if: always()
+      with:
+        name: failures-artifacts
+        path: artifacts
+

packages/e2e-tests/config/setup-debug-artifacts.js

@@ -0,0 +1,54 @@
+/**
+ * External dependencies
+ */
+const fs = require( 'fs' );
+const util = require( 'util' );
+const root = process.env.GITHUB_WORKSPACE || __dirname + '/../../../';
+const ARTIFACTS_PATH = root + '/artifacts';
+
+const writeFile = util.promisify( fs.writeFile );
+
+if ( ! fs.existsSync( ARTIFACTS_PATH ) ) {
+	fs.mkdirSync( ARTIFACTS_PATH );
+}
+
+/**
+ * Gutenberg uses the default jest-jasmine2 test runner that comes with Jest.
+ * Unfortunately, version 2 of jasmine doesn't support async reporters. It
+ * does support async before and after hooks though,  so the workaround here
+ * works by making each test wait for the artifacts before starting.
+ *
+ * Kudos to Tom Esterez (@testerez) for sharing this idea in https://github.com/smooth-code/jest-puppeteer/issues/131#issuecomment-424073620
+ */
+let artifactsPromise;
+// eslint-disable-next-line jest/no-jasmine-globals
+jasmine.getEnv().addReporter( {
+	specDone: ( result ) => {
+		if ( result.status === 'failed' ) {
+			artifactsPromise = storeArtifacts( result.fullName );
+		}
+	},
+} );
+
+beforeEach( () => artifactsPromise );
+afterAll( () => artifactsPromise );
+
+async function storeArtifacts( testName ) {
+	const slug = slugify( testName );
+	await writeFile(
+		`${ ARTIFACTS_PATH }/${ slug }-snapshot.html`,
+		await page.content()
+	);
+	await page.screenshot( { path: `${ ARTIFACTS_PATH }/${ slug }.jpg` } );
+}
+
+function slugify( testName ) {
+	const datetime = new Date().toISOString().split( '.' )[ 0 ];
+	const readableName = `${ testName } ${ datetime }`;
+	const slug = readableName
+		.toLowerCase()
+		.replace( /:/g, '-' )
+		.replace( /[^0-9a-zA-Z \-\(\)]/g, '' )
+		.replace( / /g, '-' );
+	return slug;
+}

packages/e2e-tests/jest.config.js

@@ -2,6 +2,7 @@ module.exports = {
 	...require( '@wordpress/scripts/config/jest-e2e.config' ),
 	setupFiles: [ '<rootDir>/config/gutenberg-phase.js' ],
 	setupFilesAfterEnv: [
+		'<rootDir>/config/setup-debug-artifacts.js',
 		'<rootDir>/config/setup-test-framework.js',
 		'@wordpress/jest-console',
 		'@wordpress/jest-puppeteer-axe',