diff --git a/.cirrus.yml b/.cirrus.yml index 650f55ecf18e2..23aab37308e1f 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -1,5 +1,9 @@ env: DISPLAY: :99.0 + FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992] + FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME} + FLAKINESS_DASHBOARD_BUILD_SHA: ${CIRRUS_CHANGE_IN_REPO} + FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/build/${CIRRUS_BUILD_ID} task: matrix: diff --git a/test/test.js b/test/test.js index ecb9b38d344af..0b6853b94ca00 100644 --- a/test/test.js +++ b/test/test.js @@ -111,4 +111,7 @@ new Reporter(testRunner, { projectFolder: utils.projectRoot(), showSlowTests: process.env.CI ? 5 : 0, }); + +utils.initializeFlakinessDashboardIfNeeded(testRunner); testRunner.run(); + diff --git a/test/utils.js b/test/utils.js index 05b9b7ff1cacc..71f693d3df78a 100644 --- a/test/utils.js +++ b/test/utils.js @@ -16,6 +16,7 @@ const fs = require('fs'); const path = require('path'); +const {FlakinessDashboard} = require('../utils/flakiness-dashboard'); const PROJECT_ROOT = fs.existsSync(path.join(__dirname, '..', 'package.json')) ? path.join(__dirname, '..') : path.join(__dirname, '..', '..'); /** @@ -153,4 +154,70 @@ const utils = module.exports = { }); }); }, + + initializeFlakinessDashboardIfNeeded: function(testRunner) { + // Generate testIDs for all tests and verify they don't clash. + // This will add |test.testId| for every test. + // + // NOTE: we do this unconditionally so that developers can see problems in + // their local setups. + generateTestIDs(testRunner); + // FLAKINESS_DASHBOARD_PASSWORD is an encrypted/secured variable. + // Encrypted variables get a special treatment in CI's when handling PRs so that + // secrets are not leaked to untrusted code. + // - AppVeyor DOES NOT decrypt secured variables for PRs + // - Travis DOES NOT decrypt encrypted variables for PRs + // - Cirrus CI DOES NOT decrypt encrypted variables for PRs *unless* PR is sent + // from someone who has WRITE ACCESS to the repo. + // + // Since we don't want to run flakiness dashboard for PRs on all CIs, we + // check existance of FLAKINESS_DASHBOARD_PASSWORD and absense of + // CIRRUS_BASE_SHA env variables. + if (!process.env.FLAKINESS_DASHBOARD_PASSWORD || process.env.CIRRUS_BASE_SHA) + return; + const sha = process.env.FLAKINESS_DASHBOARD_BUILD_SHA; + const dashboard = new FlakinessDashboard({ + dashboardName: process.env.FLAKINESS_DASHBOARD_NAME, + build: { + url: process.env.FLAKINESS_DASHBOARD_BUILD_URL, + name: sha.substring(0, 8), + }, + dashboardRepo: { + url: 'https://github.com/aslushnikov/puppeteer-flakiness-dashboard.git', + username: 'puppeteer-flakiness', + email: 'aslushnikov+puppeteerflakiness@gmail.com', + password: process.env.FLAKINESS_DASHBOARD_PASSWORD, + }, + }); + + testRunner.on('testfinished', test => { + const testpath = test.location.filePath.substring(utils.projectRoot().length); + const url = `https://github.com/GoogleChrome/puppeteer/blob/${sha}/${testpath}#L${test.location.lineNumber}`; + dashboard.reportTestResult({ + testId: test.testId, + name: test.location.fileName + ':' + test.location.lineNumber, + description: test.fullName, + url, + result: test.result, + }); + }); + testRunner.on('terminated', () => dashboard.uploadAndCleanup()); + testRunner.on('finished', () => dashboard.uploadAndCleanup()); + + function generateTestIDs(testRunner) { + const testIds = new Map(); + for (const test of testRunner.tests()) { + const testIdComponents = [test.name]; + for (let suite = test.suite; !!suite.parentSuite; suite = suite.parentSuite) + testIdComponents.push(suite.name); + testIdComponents.reverse(); + const testId = testIdComponents.join('>'); + const clashingTest = testIds.get(testId); + if (clashingTest) + throw new Error(`Two tests with clashing IDs: ${test.location.fileName}:${test.location.lineNumber} and ${clashingTest.location.fileName}:${clashingTest.location.lineNumber}`); + testIds.set(testId, test); + test.testId = testId; + } + } + }, }; diff --git a/utils/flakiness-dashboard/FlakinessDashboard.js b/utils/flakiness-dashboard/FlakinessDashboard.js new file mode 100644 index 0000000000000..0d8e380749cc1 --- /dev/null +++ b/utils/flakiness-dashboard/FlakinessDashboard.js @@ -0,0 +1,270 @@ +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const spawn = require('child_process').spawn; +const debug = require('debug')('flakiness'); + +const rmAsync = promisify(require('rimraf')); +const mkdtempAsync = promisify(fs.mkdtemp); +const readFileAsync = promisify(fs.readFile); +const writeFileAsync = promisify(fs.writeFile); + +const TMP_FOLDER = path.join(os.tmpdir(), 'flakiness_tmp_folder-'); + +const RED_COLOR = '\x1b[31m'; +const GREEN_COLOR = '\x1b[32m'; +const YELLOW_COLOR = '\x1b[33m'; +const RESET_COLOR = '\x1b[0m'; + +class FlakinessDashboard { + constructor({dashboardName, build, dashboardRepo, options}) { + this._dashboardName = dashboardName; + this._dashboardRepo = dashboardRepo; + this._options = options; + this._build = new Build(Date.now(), build.name, build.url, []); + } + + reportTestResult(test) { + this._build.reportTestResult(test); + } + + async uploadAndCleanup() { + console.log(`\n${YELLOW_COLOR}=== UPLOADING Flakiness Dashboard${RESET_COLOR}`); + const startTimestamp = Date.now(); + const branch = this._dashboardRepo.branch || this._dashboardName.trim().toLowerCase().replace(/\s/g, '-').replace(/[^-0-9a-zа-яё]/ig, ''); + const git = await Git.initialize(this._dashboardRepo.url, branch, this._dashboardRepo.username, this._dashboardRepo.email, this._dashboardRepo.password); + console.log(` > Dashboard Location: ${git.path()}`); + + // Do at max 5 attempts to upload changes to github. + let success = false; + const MAX_ATTEMPTS = 7; + for (let i = 0; !success && i < MAX_ATTEMPTS; ++i) { + const dashboard = await Dashboard.create(this._dashboardName, git.path(), this._options); + dashboard.addBuild(this._build); + await dashboard.saveJSON(); + await dashboard.generateReadme(); + // if push went through - great! We're done! + if (await git.commitAllAndPush()) { + success = true; + console.log(` > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${GREEN_COLOR}SUCCESS${RESET_COLOR}`); + } else { + // Otherwise - wait random time between 3 and 11 seconds. + const cooldown = 3000 + Math.round(Math.random() * 1000) * 8; + console.log(` > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${RED_COLOR}FAILED${RESET_COLOR}, cooldown ${YELLOW_COLOR}${cooldown / 1000}${RESET_COLOR} seconds`); + await new Promise(x => setTimeout(x, cooldown)); + // Reset our generated dashboard and pull from origin. + await git.hardResetToOriginMaster(); + await git.pullFromOrigin(); + } + } + await rmAsync(git.path()); + console.log(` > TOTAL TIME: ${YELLOW_COLOR}${(Date.now() - startTimestamp) / 1000}${RESET_COLOR} seconds`); + if (success) + console.log(`${YELLOW_COLOR}=== COMPLETE${RESET_COLOR}`); + else + console.log(`${RED_COLOR}=== FAILED${RESET_COLOR}`); + console.log(''); + } +} + +const DASHBOARD_VERSION = 1; +const DASHBOARD_FILENAME = 'dashboard.json'; + +class Dashboard { + static async create(name, dashboardPath, options = {}) { + const filePath = path.join(dashboardPath, DASHBOARD_FILENAME); + let data = null; + try { + data = JSON.parse(await readFileAsync(filePath)); + } catch (e) { + // Looks like there's no dashboard yet - create one. + return new Dashboard(name, dashboardPath, [], options); + } + if (!data.version) + throw new Error('cannot parse dashboard data: missing "version" field!'); + if (data.version > DASHBOARD_VERSION) + throw new Error('cannot manage dashboards that are newer then this'); + const builds = data.builds.map(build => new Build(build.timestamp, build.name, build.url, build.tests)); + return new Dashboard(name, dashboardPath, builds, options); + } + + async saveJSON() { + const data = { version: DASHBOARD_VERSION }; + data.builds = this._builds.map(build => ({ + timestamp: build._timestamp, + name: build._name, + url: build._url, + tests: build._tests, + })); + await writeFileAsync(path.join(this._dashboardPath, DASHBOARD_FILENAME), JSON.stringify(data, null, 2)); + } + + async generateReadme() { + const flakyTests = new Map(); + for (const build of this._builds) { + for (const test of build._tests) { + if (test.result !== 'ok') + flakyTests.set(test.testId, test); + } + } + + const text = []; + text.push(`# ${this._name}`); + text.push(``); + + for (const [testId, test] of flakyTests) { + text.push(`#### [${test.name}](${test.url}) - ${test.description}`); + text.push(''); + + let headers = '|'; + let splitters = '|'; + let dataColumns = '|'; + for (let i = this._builds.length - 1; i >= 0; --i) { + const build = this._builds[i]; + headers += ` [${build._name}](${build._url}) |`; + splitters += ' :---: |'; + const test = build._testsMap.get(testId); + if (test) { + const r = test.result.toLowerCase(); + let text = r; + if (r === 'ok') + text = '✅'; + else if (r.includes('fail')) + text = '🛑'; + dataColumns += ` [${text}](${test.url}) |`; + } else { + dataColumns += ` missing |`; + } + } + text.push(headers); + text.push(splitters); + text.push(dataColumns); + text.push(''); + } + + await writeFileAsync(path.join(this._dashboardPath, 'README.md'), text.join('\n')); + } + + constructor(name, dashboardPath, builds, options) { + const { + maxBuilds = 30, + } = options; + this._name = name; + this._dashboardPath = dashboardPath; + this._builds = builds.slice(builds.length - maxBuilds); + } + + addBuild(build) { + this._builds.push(build); + } +} + +class Build { + constructor(timestamp, name, url, tests) { + this._timestamp = timestamp; + this._name = name; + this._url = url; + this._tests = tests; + this._testsMap = new Map(); + for (const test of tests) + this._testsMap.set(test.testId, test); + } + + reportTestResult(test) { + this._tests.push(test); + this._testsMap.set(test.testId, test); + } +} + +module.exports = {FlakinessDashboard}; + +function promisify(nodeFunction) { + function promisified(...args) { + return new Promise((resolve, reject) => { + function callback(err, ...result) { + if (err) + return reject(err); + if (result.length === 1) + return resolve(result[0]); + return resolve(result); + } + nodeFunction.call(null, ...args, callback); + }); + } + return promisified; +} + +class Git { + static async initialize(url, branch, username, email, password) { + let schemeIndex = url.indexOf('://'); + if (schemeIndex === -1) + throw new Error(`Malformed URL "${url}": expected to start with "https://"`); + schemeIndex += '://'.length; + url = url.substring(0, schemeIndex) + username + ':' + password + '@' + url.substring(schemeIndex); + const repoPath = await mkdtempAsync(TMP_FOLDER); + // Check existance of a remote branch for this bot. + const {stdout} = await spawnAsync('git', 'ls-remote', '--heads', url, branch); + // If there is no remote branch for this bot - create one. + if (!stdout.includes(branch)) { + await spawnAsyncOrDie('git', 'clone', '--no-checkout', '--depth=1', url, repoPath); + + await spawnAsyncOrDie('git', 'checkout', '--orphan', branch, {cwd: repoPath}); + await spawnAsyncOrDie('git', 'reset', '--hard', {cwd: repoPath}); + } else { + await spawnAsyncOrDie('git', 'clone', '--single-branch', '--branch', `${branch}`, '--depth=1', url, repoPath); + } + await spawnAsyncOrDie('git', 'config', 'user.email', `"${email}"`, {cwd: repoPath}); + await spawnAsyncOrDie('git', 'config', 'user.name', `"${username}"`, {cwd: repoPath}); + return new Git(repoPath, url, branch, username); + } + + async commitAllAndPush() { + await spawnAsyncOrDie('git', 'add', '.', {cwd: this._repoPath}); + await spawnAsyncOrDie('git', 'commit', '-m', '"update dashboard"', '--author', '"puppeteer-flakiness "', {cwd: this._repoPath}); + const {code} = await spawnAsync('git', 'push', 'origin', this._branch, {cwd: this._repoPath}); + return code === 0; + } + + async hardResetToOriginMaster() { + await spawnAsyncOrDie('git', 'reset', '--hard', `origin/${this._branch}`, {cwd: this._repoPath}); + } + + async pullFromOrigin() { + await spawnAsyncOrDie('git', 'pull', 'origin', this._branch, {cwd: this._repoPath}); + } + + constructor(repoPath, url, branch, username) { + this._repoPath = repoPath; + this._url = url; + this._branch = branch; + this._username = username; + } + + path() { + return this._repoPath; + } +} + +async function spawnAsync(command, ...args) { + let options = {}; + if (args.length && args[args.length - 1].constructor.name !== 'String') + options = args.pop(); + const cmd = spawn(command, args, options); + let stdout = ''; + let stderr = ''; + cmd.stdout.on('data', data => stdout += data); + cmd.stderr.on('data', data => stderr += data); + const code = await new Promise(x => cmd.once('close', x)); + if (stdout) + debug(stdout); + if (stderr) + debug(stderr); + return {code, stdout, stderr}; +} + +async function spawnAsyncOrDie(command, ...args) { + const {code, stdout, stderr} = await spawnAsync(command, ...args); + if (code !== 0) + throw new Error(`Failed to executed: "${command} ${args.join(' ')}".\n\n=== STDOUT ===\n${stdout}\n\n\n=== STDERR ===\n${stderr}`); + return {stdout, stderr}; +} diff --git a/utils/flakiness-dashboard/index.js b/utils/flakiness-dashboard/index.js new file mode 100644 index 0000000000000..bb6ec923fa797 --- /dev/null +++ b/utils/flakiness-dashboard/index.js @@ -0,0 +1,3 @@ +const {FlakinessDashboard} = require('./FlakinessDashboard'); + +module.exports = {FlakinessDashboard}; diff --git a/utils/node6-transform/index.js b/utils/node6-transform/index.js index 967899f9a104d..e96a79cf33443 100644 --- a/utils/node6-transform/index.js +++ b/utils/node6-transform/index.js @@ -35,6 +35,7 @@ copyFolder(path.join(root, 'lib'), path.join(dest, 'lib')); copyFolder(path.join(root, 'test'), path.join(dest, 'test')); copyFolder(path.join(root, 'utils', 'testrunner'), path.join(dest, 'utils', 'testrunner')); copyFolder(path.join(root, 'utils', 'testserver'), path.join(dest, 'utils', 'testserver')); +copyFolder(path.join(root, 'utils', 'flakiness-dashboard'), path.join(dest, 'utils', 'flakiness-dashboard')); function copyFolder(source, target) { if (fs.existsSync(target))