Skip to content

Commit

Permalink
Store screenshots of CI e2e failures as CI artifacts (#26664)
Browse files Browse the repository at this point in the history
* squash

* Remove artifacts

* Rename screenshots to artifacts

* remove usages

* fix broken test
  • Loading branch information
adamziel committed Nov 12, 2020
1 parent b0d5430 commit 981b3c8
Show file tree
Hide file tree
Showing 3 changed files with 84 additions and 0 deletions.
29 changes: 29 additions & 0 deletions .github/workflows/end2end-test.yml
Expand Up @@ -51,6 +51,13 @@ jobs:
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 0' < ~/.jest-e2e-tests )
- name: Archive debug artifacts (screenshots, HTML snapshots)
uses: actions/upload-artifact@v2
if: always()
with:
name: failures-artifacts
path: artifacts

admin-2:
name: Admin - 2

Expand Down Expand Up @@ -92,6 +99,13 @@ jobs:
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 1' < ~/.jest-e2e-tests )
- name: Archive debug artifacts (screenshots, HTML snapshots)
uses: actions/upload-artifact@v2
if: always()
with:
name: failures-artifacts
path: artifacts

admin-3:
name: Admin - 3

Expand Down Expand Up @@ -133,6 +147,13 @@ jobs:
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 2' < ~/.jest-e2e-tests )
- name: Archive debug artifacts (screenshots, HTML snapshots)
uses: actions/upload-artifact@v2
if: always()
with:
name: failures-artifacts
path: artifacts

admin-4:
name: Admin - 4

Expand Down Expand Up @@ -173,3 +194,11 @@ jobs:
run: |
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --listTests > ~/.jest-e2e-tests
$( npm bin )/wp-scripts test-e2e --config=./packages/e2e-tests/jest.config.js --cacheDirectory="$HOME/.jest-cache" --runTestsByPath $( awk 'NR % 4 == 3' < ~/.jest-e2e-tests )
- name: Archive debug artifacts (screenshots, HTML snapshots)
uses: actions/upload-artifact@v2
if: always()
with:
name: failures-artifacts
path: artifacts

54 changes: 54 additions & 0 deletions packages/e2e-tests/config/setup-debug-artifacts.js
@@ -0,0 +1,54 @@
/**
* External dependencies
*/
const fs = require( 'fs' );
const util = require( 'util' );
const root = process.env.GITHUB_WORKSPACE || __dirname + '/../../../';
const ARTIFACTS_PATH = root + '/artifacts';

const writeFile = util.promisify( fs.writeFile );

if ( ! fs.existsSync( ARTIFACTS_PATH ) ) {
fs.mkdirSync( ARTIFACTS_PATH );
}

/**
* Gutenberg uses the default jest-jasmine2 test runner that comes with Jest.
* Unfortunately, version 2 of jasmine doesn't support async reporters. It
* does support async before and after hooks though, so the workaround here
* works by making each test wait for the artifacts before starting.
*
* Kudos to Tom Esterez (@testerez) for sharing this idea in https://github.com/smooth-code/jest-puppeteer/issues/131#issuecomment-424073620
*/
let artifactsPromise;
// eslint-disable-next-line jest/no-jasmine-globals
jasmine.getEnv().addReporter( {
specDone: ( result ) => {
if ( result.status === 'failed' ) {
artifactsPromise = storeArtifacts( result.fullName );
}
},
} );

beforeEach( () => artifactsPromise );
afterAll( () => artifactsPromise );

async function storeArtifacts( testName ) {
const slug = slugify( testName );
await writeFile(
`${ ARTIFACTS_PATH }/${ slug }-snapshot.html`,
await page.content()
);
await page.screenshot( { path: `${ ARTIFACTS_PATH }/${ slug }.jpg` } );
}

function slugify( testName ) {
const datetime = new Date().toISOString().split( '.' )[ 0 ];
const readableName = `${ testName } ${ datetime }`;
const slug = readableName
.toLowerCase()
.replace( /:/g, '-' )
.replace( /[^0-9a-zA-Z \-\(\)]/g, '' )
.replace( / /g, '-' );
return slug;
}
1 change: 1 addition & 0 deletions packages/e2e-tests/jest.config.js
Expand Up @@ -2,6 +2,7 @@ module.exports = {
...require( '@wordpress/scripts/config/jest-e2e.config' ),
setupFiles: [ '<rootDir>/config/gutenberg-phase.js' ],
setupFilesAfterEnv: [
'<rootDir>/config/setup-debug-artifacts.js',
'<rootDir>/config/setup-test-framework.js',
'@wordpress/jest-console',
'@wordpress/jest-puppeteer-axe',
Expand Down

0 comments on commit 981b3c8

Please sign in to comment.