Skip to content

Commit

Permalink
Report performance in CI (#5488)
Browse files Browse the repository at this point in the history
* Report performance in CI

* Include the report of executing perf-report/index.js

* Remove all ANSI styles

* Report added or removed tree-shaking stages
  • Loading branch information
TrickyPi committed Apr 27, 2024
1 parent db19272 commit 82ec9b4
Show file tree
Hide file tree
Showing 5 changed files with 265 additions and 18 deletions.
151 changes: 151 additions & 0 deletions .github/workflows/performance-report.yml
@@ -0,0 +1,151 @@
name: Performance Report
env:
BUILD_BOOTSTRAP_CJS: mv dist dist-build && node dist-build/bin/rollup --config rollup.config.ts --configPlugin typescript --configTest --forceExit && rm -rf dist-build

on:
pull_request:
types:
- synchronize
- opened
- reopened

concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true

permissions:
contents: read

jobs:
build-artefacts:
strategy:
matrix:
settings:
- name: current
ref: refs/pull/${{ github.event.number }}/merge
- name: previous
ref: ${{github.event.pull_request.base.ref}}
name: Build ${{matrix.settings.name}} artefact
runs-on: ubuntu-latest
steps:
- name: Checkout Commit
uses: actions/checkout@v4
with:
ref: ${{matrix.settings.ref}}
- name: Install Toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: nightly-2023-10-05
targets: x86_64-unknown-linux-gnu
- name: Cache cargo
uses: actions/cache@v4
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
.cargo-cache
rust/target/
key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }}
restore-keys: cargo-cache
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 18
- name: Cache Node Modules
id: cache-node-modules
uses: actions/cache@v4
with:
path: node_modules
key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci --ignore-scripts
- name: Build artefacts 123
run: npm exec -- concurrently -c green,blue 'npm:build:napi -- --release' 'npm:build:cjs' && npm run build:copy-native && ${{env.BUILD_BOOTSTRAP_CJS}} && npm run build:copy-native
- name: Upload artifact
uses: actions/upload-artifact@v4
with:
name: ${{ matrix.settings.name }}
path: dist/
if-no-files-found: error

report:
needs: build-artefacts
permissions:
pull-requests: write
runs-on: ubuntu-latest
name: Report Performance
steps:
- name: Checkout Commit
uses: actions/checkout@v4
with:
ref: refs/pull/${{ github.event.number }}/merge
- name: Install Toolchain
uses: dtolnay/rust-toolchain@stable
with:
toolchain: nightly-2023-10-05
targets: x86_64-unknown-linux-gnu
- name: Cache cargo
uses: actions/cache@v4
with:
path: |
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
.cargo-cache
rust/target/
key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }}
restore-keys: cargo-cache
- name: Setup Node
uses: actions/setup-node@v4
with:
node-version: 18
- name: Cache Node Modules
id: cache-node-modules
uses: actions/cache@v4
with:
path: node_modules
key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci --ignore-scripts
- name: Download all artifacts
uses: actions/download-artifact@v4
with:
path: _benchmark
- name: Change rollup import in internal benchmark
run: |
echo "export { rollup as previousRollup, VERSION as previousVersion } from '../../_benchmark/previous/rollup.js';" > ./scripts/perf-report/rollup-artefacts.js
echo "export { rollup as newRollup } from '../../_benchmark/current/rollup.js';" >> ./scripts/perf-report/rollup-artefacts.js
- name: Run internal benchmark
run: node --expose-gc scripts/perf-report/index.js
- name: Install benchmark tool
run: cargo install --locked hyperfine
- name: Run Rough benchmark
run: |
hyperfine --warmup 1 --export-markdown _benchmark/rough-report.md --show-output --runs 3 \
'node _benchmark/previous/bin/rollup -i ./perf/entry.js -o _benchmark/result/previous.js' \
'node _benchmark/current/bin/rollup -i ./perf/entry.js -o _benchmark/result/current.js'
- name: Combine bechmark reports
run: |
echo "# Performance report!" > _benchmark/result.md
echo "## Rough benchmark" >> _benchmark/result.md
cat _benchmark/rough-report.md >> _benchmark/result.md
echo "## Internal benchmark" >> _benchmark/result.md
cat _benchmark/internal-report.md >> _benchmark/result.md
- name: Find Performance report
uses: peter-evans/find-comment@v3
id: findPerformanceReport
with:
issue-number: ${{ github.event.number }}
comment-author: 'github-actions[bot]'
body-includes: 'Performance report'
- name: Create or update Performance report
uses: peter-evans/create-or-update-comment@v4
id: createOrUpdatePerformanceReport
with:
comment-id: ${{ steps.findPerformanceReport.outputs.comment-id }}
issue-number: ${{ github.event.number }}
edit-mode: replace
body-path: _benchmark/result.md
2 changes: 1 addition & 1 deletion package.json
Expand Up @@ -66,7 +66,7 @@
"lint:markdown:nofix": "prettier --check \"**/*.md\"",
"lint:rust": "cd rust && cargo fmt && cargo clippy --fix --allow-dirty",
"lint:rust:nofix": "cd rust && cargo fmt --check && cargo clippy",
"perf": "npm run build && node --expose-gc scripts/perf.js",
"perf": "npm run build && node --expose-gc scripts/perf-report/index.js",
"prepare": "husky && node scripts/check-release.js || npm run build:prepare",
"prepublishOnly": "node scripts/check-release.js && node scripts/prepublish.js",
"postpublish": "node scripts/postpublish.js",
Expand Down
80 changes: 63 additions & 17 deletions scripts/perf.js → scripts/perf-report/index.js
Expand Up @@ -5,10 +5,9 @@ import { chdir } from 'node:process';
import { fileURLToPath } from 'node:url';
import { createColors } from 'colorette';
import prettyBytes from 'pretty-bytes';
import { rollup as previousRollup, VERSION as previousVersion } from 'rollup';
// eslint-disable-next-line import/no-unresolved
import { rollup as newRollup } from '../dist/rollup.js';
import { runWithEcho } from './helpers.js';
import { runWithEcho } from '../helpers.js';
import reportCollector from './report-collector.js';
import { newRollup, previousRollup, previousVersion } from './rollup-artefacts.js';

/**
* @typedef {Record<string,{memory:number,time:number}>} CollectedTimings
Expand All @@ -18,7 +17,7 @@ import { runWithEcho } from './helpers.js';
* @typedef {Record<string, [number, number, number][]>} AccumulatedTimings
*/

const PERF_DIRECTORY = new URL('../perf/', import.meta.url);
const PERF_DIRECTORY = new URL('../../perf/', import.meta.url);
const ENTRY = new URL('entry.js', PERF_DIRECTORY);
const THREEJS_COPIES = 10;
const { bold, underline, cyan, red, green } = createColors();
Expand Down Expand Up @@ -91,10 +90,12 @@ async function calculatePrintAndPersistTimings() {
);
clearLines(numberOfLinesToClear);
}
reportCollector.startRecord();
printMeasurements(
getAverage(accumulatedNewTimings, RUNS_TO_AVERAGE),
getAverage(accumulatedPreviousTimings, RUNS_TO_AVERAGE)
);
await reportCollector.outputMsg();
}

/**
Expand Down Expand Up @@ -172,9 +173,16 @@ function getSingleAverage(times, runs, discarded) {
* @return {number}
*/
function printMeasurements(newAverage, previousAverage, filter = /.*/) {
const printedLabels = Object.keys(newAverage).filter(label => filter.test(label));
console.info('');
for (const label of printedLabels) {
const newPrintedLabels = Object.keys(newAverage).filter(predicateLabel);
const previousPrintedLabels = Object.keys(previousAverage).filter(predicateLabel);

const newTreeShakings = newPrintedLabels.filter(isTreeShakingLabel);
const oldTreeShakings = previousPrintedLabels.filter(isTreeShakingLabel);

const addedTreeShaking = newTreeShakings.length - oldTreeShakings.length;
let treeShakingCount = 0;

for (const label of newPrintedLabels) {
/**
* @type {function(string): string}
*/
Expand All @@ -185,16 +193,54 @@ function printMeasurements(newAverage, previousAverage, filter = /.*/) {
color = underline;
}
}
console.info(
color(
`${label}: ${getFormattedTime(
newAverage[label].time,
previousAverage[label]?.time
)}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}`
)
);
const texts = [];
if (isTreeShakingLabel(label)) {
treeShakingCount++;
if (addedTreeShaking < 0 && treeShakingCount === newTreeShakings.length) {
texts.push(generateSingleReport(label));
for (const label of oldTreeShakings.slice(addedTreeShaking)) {
const { time, memory } = previousAverage[label];
texts.push(`${label}: ${time.toFixed(0)}ms, ${prettyBytes(memory)}, removed stage`);
}
} else if (addedTreeShaking > 0 && treeShakingCount > oldTreeShakings.length) {
texts.push(generateSingleReport(label, ', new stage'));
} else {
texts.push(generateSingleReport(label));
}
} else {
texts.push(generateSingleReport(label));
}
for (const text of texts) {
reportCollector.push(text);
console.info(color(text));
}
}
return Math.max(newPrintedLabels.length, previousPrintedLabels.length) + 2;

/**
* @param {string} label
*/
function predicateLabel(label) {
return filter.test(label);
}

/**
* @param {string} label
* @param {string} addon
*/
function generateSingleReport(label, addon = '') {
return `${label}: ${getFormattedTime(
newAverage[label].time,
previousAverage[label]?.time
)}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}${addon}`;
}
return printedLabels.length + 2;
}

/**
* @param {string} label
*/
function isTreeShakingLabel(label) {
return label.startsWith('treeshaking pass');
}

/**
Expand Down
47 changes: 47 additions & 0 deletions scripts/perf-report/report-collector.js
@@ -0,0 +1,47 @@
import { writeFile } from 'node:fs/promises';
import { fileURLToPath } from 'node:url';

export default new (class ReportCollector {
/**
* @type {string[]}
*/
#messageList = [];
#isRecording = false;
startRecord() {
this.#isRecording = true;
}
/**
* @param {string} message
*/
push(message) {
if (!this.#isRecording) return;
if (message.startsWith('#')) {
message = '##' + message;
}
this.#messageList.push(message);
}
outputMsg() {
if (process.env.CI) {
return writeFile(
fileURLToPath(new URL('../../_benchmark/internal-report.md', import.meta.url)),
removeAnsiStyles(this.#messageList.join('\n'))
);
}
}
})();

/**
* @param {string} text
* @returns {string}
*/
function removeAnsiStyles(text) {
const ansiRegex = new RegExp(
[
'[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\\u0007)',
'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))'
].join('|'),
'g'
);

return text.replace(ansiRegex, '');
}
3 changes: 3 additions & 0 deletions scripts/perf-report/rollup-artefacts.js
@@ -0,0 +1,3 @@
export { rollup as previousRollup, VERSION as previousVersion } from 'rollup';
// eslint-disable-next-line import/no-unresolved
export { rollup as newRollup } from '../../dist/rollup.js';

0 comments on commit 82ec9b4

Please sign in to comment.