From e252dcf200cb0a61fe268af674cb12542bf436bc Mon Sep 17 00:00:00 2001 From: Andrey Lushnikov Date: Thu, 1 Aug 2019 16:09:02 -0700 Subject: [PATCH] chore(flakiness): Flakiness Dashboard fixes (#4788) - fix `FLAKINESS_DASHBOARD_BUILD_URL` to point to a task instead of a build - do not pretty-print `dashboard.json` when serializing flakiness results - filter out 'COVERAGE' test(s) so that they don't add up to `dashboard.json` payload. These are useless - validate certain important options of flakiness dashboard - more logging to STDOUT to actually say which repo and what branch is getting used - enhance commit message with a build URL - use a more compact format for JSON. For 100 runs of 700 tests it yields 21MB json instead of 23MB. - bump default builds number to 100 --- .cirrus.yml | 2 +- test/utils.js | 4 ++ .../flakiness-dashboard/FlakinessDashboard.js | 44 ++++++++++++++----- 3 files changed, 38 insertions(+), 12 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 23aab37308e1f..353933cb03760 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -3,7 +3,7 @@ env: FLAKINESS_DASHBOARD_PASSWORD: ENCRYPTED[b3e207db5d153b543f219d3c3b9123d8321834b783b9e45ac7d380e026ab3a56398bde51b521ac5859e7e45cb95d0992] FLAKINESS_DASHBOARD_NAME: Cirrus ${CIRRUS_TASK_NAME} FLAKINESS_DASHBOARD_BUILD_SHA: ${CIRRUS_CHANGE_IN_REPO} - FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/build/${CIRRUS_BUILD_ID} + FLAKINESS_DASHBOARD_BUILD_URL: https://cirrus-ci.com/task/${CIRRUS_TASK_ID} task: matrix: diff --git a/test/utils.js b/test/utils.js index 71f693d3df78a..fbb95dde2295e 100644 --- a/test/utils.js +++ b/test/utils.js @@ -191,6 +191,10 @@ const utils = module.exports = { }); testRunner.on('testfinished', test => { + // Do not report tests from COVERAGE testsuite. + // They don't bring much value to us. + if (test.fullName.startsWith('COVERAGE')) + return; const testpath = test.location.filePath.substring(utils.projectRoot().length); const url = `https://github.com/GoogleChrome/puppeteer/blob/${sha}/${testpath}#L${test.location.lineNumber}`; dashboard.reportTestResult({ diff --git a/utils/flakiness-dashboard/FlakinessDashboard.js b/utils/flakiness-dashboard/FlakinessDashboard.js index 0d8e380749cc1..d0fc035c9b201 100644 --- a/utils/flakiness-dashboard/FlakinessDashboard.js +++ b/utils/flakiness-dashboard/FlakinessDashboard.js @@ -18,6 +18,14 @@ const RESET_COLOR = '\x1b[0m'; class FlakinessDashboard { constructor({dashboardName, build, dashboardRepo, options}) { + if (!dashboardName) + throw new Error('"options.dashboardName" must be specified!'); + if (!build) + throw new Error('"options.build" must be specified!'); + if (!build.url) + throw new Error('"options.build.url" must be specified!'); + if (!build.name) + throw new Error('"options.build.name" must be specified!'); this._dashboardName = dashboardName; this._dashboardRepo = dashboardRepo; this._options = options; @@ -32,8 +40,10 @@ class FlakinessDashboard { console.log(`\n${YELLOW_COLOR}=== UPLOADING Flakiness Dashboard${RESET_COLOR}`); const startTimestamp = Date.now(); const branch = this._dashboardRepo.branch || this._dashboardName.trim().toLowerCase().replace(/\s/g, '-').replace(/[^-0-9a-zа-яё]/ig, ''); + console.log(` > Dashboard URL: ${this._dashboardRepo.url}`); + console.log(` > Dashboard Branch: ${branch}`); const git = await Git.initialize(this._dashboardRepo.url, branch, this._dashboardRepo.username, this._dashboardRepo.email, this._dashboardRepo.password); - console.log(` > Dashboard Location: ${git.path()}`); + console.log(` > Dashboard Checkout: ${git.path()}`); // Do at max 5 attempts to upload changes to github. let success = false; @@ -44,7 +54,7 @@ class FlakinessDashboard { await dashboard.saveJSON(); await dashboard.generateReadme(); // if push went through - great! We're done! - if (await git.commitAllAndPush()) { + if (await git.commitAllAndPush(`update dashboard\n\nbuild: ${this._build._url}`)) { success = true; console.log(` > Push attempt ${YELLOW_COLOR}${i + 1}${RESET_COLOR} of ${YELLOW_COLOR}${MAX_ATTEMPTS}${RESET_COLOR}: ${GREEN_COLOR}SUCCESS${RESET_COLOR}`); } else { @@ -84,19 +94,31 @@ class Dashboard { throw new Error('cannot parse dashboard data: missing "version" field!'); if (data.version > DASHBOARD_VERSION) throw new Error('cannot manage dashboards that are newer then this'); - const builds = data.builds.map(build => new Build(build.timestamp, build.name, build.url, build.tests)); + const builds = data.builds.map(build => new Build(build.ts, build.n, build.u, build.t.map(test => ({ + testId: test.i, + name: test.n, + description: test.d, + url: test.u, + result: test.r, + })))); return new Dashboard(name, dashboardPath, builds, options); } async saveJSON() { const data = { version: DASHBOARD_VERSION }; data.builds = this._builds.map(build => ({ - timestamp: build._timestamp, - name: build._name, - url: build._url, - tests: build._tests, + ts: build._timestamp, + n: build._name, + u: build._url, + t: build._tests.map(test => ({ + i: test.testId, + n: test.name, + d: test.description, + u: test.url, + r: test.result, + })), })); - await writeFileAsync(path.join(this._dashboardPath, DASHBOARD_FILENAME), JSON.stringify(data, null, 2)); + await writeFileAsync(path.join(this._dashboardPath, DASHBOARD_FILENAME), JSON.stringify(data)); } async generateReadme() { @@ -147,7 +169,7 @@ class Dashboard { constructor(name, dashboardPath, builds, options) { const { - maxBuilds = 30, + maxBuilds = 100, } = options; this._name = name; this._dashboardPath = dashboardPath; @@ -218,9 +240,9 @@ class Git { return new Git(repoPath, url, branch, username); } - async commitAllAndPush() { + async commitAllAndPush(message) { await spawnAsyncOrDie('git', 'add', '.', {cwd: this._repoPath}); - await spawnAsyncOrDie('git', 'commit', '-m', '"update dashboard"', '--author', '"puppeteer-flakiness "', {cwd: this._repoPath}); + await spawnAsyncOrDie('git', 'commit', '-m', `${message}`, '--author', '"puppeteer-flakiness "', {cwd: this._repoPath}); const {code} = await spawnAsync('git', 'push', 'origin', this._branch, {cwd: this._repoPath}); return code === 0; }