From a206ee0bebb0f813d0bc585de811a560ba87347e Mon Sep 17 00:00:00 2001 From: XiaoPi <530257315@qq.com> Date: Tue, 23 Apr 2024 20:53:30 +0800 Subject: [PATCH 1/4] Report performance in CI --- .github/workflows/performance-report.yml | 140 +++++++++++++++++++++++ 1 file changed, 140 insertions(+) create mode 100644 .github/workflows/performance-report.yml diff --git a/.github/workflows/performance-report.yml b/.github/workflows/performance-report.yml new file mode 100644 index 00000000000..3c44315bdd0 --- /dev/null +++ b/.github/workflows/performance-report.yml @@ -0,0 +1,140 @@ +name: Performance Report +env: + BUILD_BOOTSTRAP_CJS: mv dist dist-build && node dist-build/bin/rollup --config rollup.config.ts --configPlugin typescript --configTest --forceExit && rm -rf dist-build + BUILD_COMMAND_OTPION: --config rollup.config.ts --configPlugin typescript --configTest --forceExit + +on: + pull_request: + types: + - synchronize + - opened + - reopened + +concurrency: + group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +permissions: + contents: read + +jobs: + build-artefacts: + strategy: + matrix: + settings: + - name: current + ref: refs/pull/${{ github.event.number }}/merge + - name: previous + ref: ${{github.event.pull_request.base.ref}} + name: Build ${{matrix.settings.name}} artefact + runs-on: ubuntu-latest + steps: + - name: Checkout Commit + uses: actions/checkout@v4 + with: + ref: ${{matrix.settings.ref}} + - name: Install Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly-2023-10-05 + targets: x86_64-unknown-linux-gnu + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + .cargo-cache + rust/target/ + key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }} + restore-keys: cargo-cache + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Cache Node Modules + id: cache-node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci --ignore-scripts + - name: Build artefacts 123 + run: npm exec -- concurrently -c green,blue 'npm:build:napi -- --release' 'npm:build:cjs' && npm run build:copy-native && ${{env.BUILD_BOOTSTRAP_CJS}} && npm run build:copy-native + - name: Upload artifact + uses: actions/upload-artifact@v4 + with: + name: ${{ matrix.settings.name }} + path: dist/ + if-no-files-found: error + + report: + needs: build-artefacts + permissions: + pull-requests: write + runs-on: ubuntu-latest + name: Report Performance + steps: + - name: Checkout Commit + uses: actions/checkout@v4 + with: + ref: ${{github.event.pull_request.base.ref}} + - name: Install Toolchain + uses: dtolnay/rust-toolchain@stable + with: + toolchain: nightly-2023-10-05 + targets: x86_64-unknown-linux-gnu + - name: Cache cargo + uses: actions/cache@v4 + with: + path: | + ~/.cargo/registry/index/ + ~/.cargo/registry/cache/ + ~/.cargo/git/db/ + .cargo-cache + rust/target/ + key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }} + restore-keys: cargo-cache + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 18 + - name: Cache Node Modules + id: cache-node-modules + uses: actions/cache@v4 + with: + path: node_modules + key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }} + - name: Install dependencies + if: steps.cache-node-modules.outputs.cache-hit != 'true' + run: npm ci --ignore-scripts + - name: Download all artifacts + uses: actions/download-artifact@v4 + with: + path: _benchmark + - name: Create benchmark directory + run: mkdir -p _benchmark + - name: Install benchmark tool + run: cargo install --locked hyperfine + - name: Run benchmark + run: hyperfine --warmup 1 --export-markdown _benchmark/result.md --show-output --runs 3 'node _benchmark/previous/bin/rollup ${{env.BUILD_COMMAND_OTPION}}' 'node _benchmark/current/bin/rollup ${{env.BUILD_COMMAND_OTPION}}' + - name: Modify benchmark result file + run: sed -i '1s;^;### Performance report!\n\n;' _benchmark/result.md + - name: Find Performance report + uses: peter-evans/find-comment@v3 + id: findPerformanceReport + with: + issue-number: ${{ github.event.number }} + comment-author: 'github-actions[bot]' + body-includes: 'Performance report' + - name: Create or update Performance report + uses: peter-evans/create-or-update-comment@v4 + id: createOrUpdatePerformanceReport + with: + comment-id: ${{ steps.findPerformanceReport.outputs.comment-id }} + issue-number: ${{ github.event.number }} + edit-mode: replace + body-path: _benchmark/result.md From 27be82ac838ee4f7c0064304fa0c400ef7d00730 Mon Sep 17 00:00:00 2001 From: XiaoPi <530257315@qq.com> Date: Thu, 25 Apr 2024 10:42:08 +0800 Subject: [PATCH 2/4] Include the report of executing perf-report/index.js --- .github/workflows/performance-report.yml | 41 ++++++++++++++--------- package.json | 2 +- scripts/{perf.js => perf-report/index.js} | 25 +++++++------- scripts/perf-report/report-collector.js | 31 +++++++++++++++++ scripts/perf-report/rollup-artefacts.js | 3 ++ 5 files changed, 73 insertions(+), 29 deletions(-) rename scripts/{perf.js => perf-report/index.js} (92%) create mode 100644 scripts/perf-report/report-collector.js create mode 100644 scripts/perf-report/rollup-artefacts.js diff --git a/.github/workflows/performance-report.yml b/.github/workflows/performance-report.yml index 3c44315bdd0..6859013c501 100644 --- a/.github/workflows/performance-report.yml +++ b/.github/workflows/performance-report.yml @@ -1,14 +1,13 @@ name: Performance Report env: - BUILD_BOOTSTRAP_CJS: mv dist dist-build && node dist-build/bin/rollup --config rollup.config.ts --configPlugin typescript --configTest --forceExit && rm -rf dist-build - BUILD_COMMAND_OTPION: --config rollup.config.ts --configPlugin typescript --configTest --forceExit + BUILD_BOOTSTRAP_CJS: mv dist dist-build && node dist-build/bin/rollup --config rollup.config.ts --configPlugin typescript --configTest --forceExit && rm -rf dist-build on: - pull_request: - types: - - synchronize - - opened - - reopened + pull_request: + types: + - synchronize + - opened + - reopened concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -28,7 +27,7 @@ jobs: ref: ${{github.event.pull_request.base.ref}} name: Build ${{matrix.settings.name}} artefact runs-on: ubuntu-latest - steps: + steps: - name: Checkout Commit uses: actions/checkout@v4 with: @@ -81,7 +80,7 @@ jobs: - name: Checkout Commit uses: actions/checkout@v4 with: - ref: ${{github.event.pull_request.base.ref}} + ref: refs/pull/${{ github.event.number }}/merge - name: Install Toolchain uses: dtolnay/rust-toolchain@stable with: @@ -115,14 +114,26 @@ jobs: uses: actions/download-artifact@v4 with: path: _benchmark - - name: Create benchmark directory - run: mkdir -p _benchmark + - name: Change rollup import in internal benchmark + run: | + echo "export { rollup as previousRollup, VERSION as previousVersion } from '../../_benchmark/previous/rollup.js';" > ./scripts/perf-report/rollup-artefacts.js + echo "export { rollup as newRollup } from '../../_benchmark/current/rollup.js';" >> ./scripts/perf-report/rollup-artefacts.js + - name: Run internal benchmark + run: node --expose-gc scripts/perf-report/index.js - name: Install benchmark tool run: cargo install --locked hyperfine - - name: Run benchmark - run: hyperfine --warmup 1 --export-markdown _benchmark/result.md --show-output --runs 3 'node _benchmark/previous/bin/rollup ${{env.BUILD_COMMAND_OTPION}}' 'node _benchmark/current/bin/rollup ${{env.BUILD_COMMAND_OTPION}}' - - name: Modify benchmark result file - run: sed -i '1s;^;### Performance report!\n\n;' _benchmark/result.md + - name: Run Rough benchmark + run: | + hyperfine --warmup 1 --export-markdown _benchmark/rough-report.md --show-output --runs 3 \ + 'node _benchmark/previous/bin/rollup -i ./perf/entry.js -o _benchmark/result/previous.js' \ + 'node _benchmark/current/bin/rollup -i ./perf/entry.js -o _benchmark/result/current.js' + - name: Combine bechmark reports + run: | + echo "# Performance report!\n" > _benchmark/result.md + echo "## Rough benchmark\n" >> _benchmark/result.md + cat _benchmark/rough-report.md >> _benchmark/result.md + echo "## Internal benchmark\n" >> _benchmark/result.md + cat _benchmark/internal-report.md >> _benchmark/result.md - name: Find Performance report uses: peter-evans/find-comment@v3 id: findPerformanceReport diff --git a/package.json b/package.json index 19438a9b95f..2137fe27b7e 100644 --- a/package.json +++ b/package.json @@ -66,7 +66,7 @@ "lint:markdown:nofix": "prettier --check \"**/*.md\"", "lint:rust": "cd rust && cargo fmt && cargo clippy --fix --allow-dirty", "lint:rust:nofix": "cd rust && cargo fmt --check && cargo clippy", - "perf": "npm run build && node --expose-gc scripts/perf.js", + "perf": "npm run build && node --expose-gc scripts/perf-report/index.js", "prepare": "husky && node scripts/check-release.js || npm run build:prepare", "prepublishOnly": "node scripts/check-release.js && node scripts/prepublish.js", "postpublish": "node scripts/postpublish.js", diff --git a/scripts/perf.js b/scripts/perf-report/index.js similarity index 92% rename from scripts/perf.js rename to scripts/perf-report/index.js index e8aba413bf6..e9b12ef0a9c 100644 --- a/scripts/perf.js +++ b/scripts/perf-report/index.js @@ -5,10 +5,9 @@ import { chdir } from 'node:process'; import { fileURLToPath } from 'node:url'; import { createColors } from 'colorette'; import prettyBytes from 'pretty-bytes'; -import { rollup as previousRollup, VERSION as previousVersion } from 'rollup'; -// eslint-disable-next-line import/no-unresolved -import { rollup as newRollup } from '../dist/rollup.js'; -import { runWithEcho } from './helpers.js'; +import { runWithEcho } from '../helpers.js'; +import reportCollector from './report-collector.js'; +import { newRollup, previousRollup, previousVersion } from './rollup-artefacts.js'; /** * @typedef {Record} CollectedTimings @@ -18,7 +17,7 @@ import { runWithEcho } from './helpers.js'; * @typedef {Record} AccumulatedTimings */ -const PERF_DIRECTORY = new URL('../perf/', import.meta.url); +const PERF_DIRECTORY = new URL('../../perf/', import.meta.url); const ENTRY = new URL('entry.js', PERF_DIRECTORY); const THREEJS_COPIES = 10; const { bold, underline, cyan, red, green } = createColors(); @@ -91,10 +90,12 @@ async function calculatePrintAndPersistTimings() { ); clearLines(numberOfLinesToClear); } + reportCollector.startRecord(); printMeasurements( getAverage(accumulatedNewTimings, RUNS_TO_AVERAGE), getAverage(accumulatedPreviousTimings, RUNS_TO_AVERAGE) ); + await reportCollector.outputMsg(); } /** @@ -185,14 +186,12 @@ function printMeasurements(newAverage, previousAverage, filter = /.*/) { color = underline; } } - console.info( - color( - `${label}: ${getFormattedTime( - newAverage[label].time, - previousAverage[label]?.time - )}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}` - ) - ); + const text = `${label}: ${getFormattedTime( + newAverage[label].time, + previousAverage[label]?.time + )}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}`; + reportCollector.push(text); + console.info(color(text)); } return printedLabels.length + 2; } diff --git a/scripts/perf-report/report-collector.js b/scripts/perf-report/report-collector.js new file mode 100644 index 00000000000..ca07bb8f3aa --- /dev/null +++ b/scripts/perf-report/report-collector.js @@ -0,0 +1,31 @@ +import { writeFile } from 'node:fs/promises'; +import { fileURLToPath } from 'node:url'; + +export default new (class ReportCollector { + /** + * @type {string[]} + */ + #messageList = []; + #isRecording = false; + startRecord() { + this.#isRecording = true; + } + /** + * @param {string} message + */ + push(message) { + if (!this.#isRecording) return; + if (message.startsWith('#')) { + message = '##' + message; + } + this.#messageList.push(message); + } + outputMsg() { + if (process.env.CI) { + return writeFile( + fileURLToPath(new URL('../../_benchmark/internal-report.md', import.meta.url)), + this.#messageList.join('\n') + ); + } + } +})(); diff --git a/scripts/perf-report/rollup-artefacts.js b/scripts/perf-report/rollup-artefacts.js new file mode 100644 index 00000000000..936e44ce426 --- /dev/null +++ b/scripts/perf-report/rollup-artefacts.js @@ -0,0 +1,3 @@ +export { rollup as previousRollup, VERSION as previousVersion } from 'rollup'; +// eslint-disable-next-line import/no-unresolved +export { rollup as newRollup } from '../../dist/rollup.js'; From 17ce1f93ecaf5582931751d98b4896415d5c0585 Mon Sep 17 00:00:00 2001 From: XiaoPi <530257315@qq.com> Date: Thu, 25 Apr 2024 11:40:09 +0800 Subject: [PATCH 3/4] Remove all ANSI styles --- .github/workflows/performance-report.yml | 6 +++--- scripts/perf-report/report-collector.js | 18 +++++++++++++++++- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/performance-report.yml b/.github/workflows/performance-report.yml index 6859013c501..6e4a8bdf611 100644 --- a/.github/workflows/performance-report.yml +++ b/.github/workflows/performance-report.yml @@ -129,10 +129,10 @@ jobs: 'node _benchmark/current/bin/rollup -i ./perf/entry.js -o _benchmark/result/current.js' - name: Combine bechmark reports run: | - echo "# Performance report!\n" > _benchmark/result.md - echo "## Rough benchmark\n" >> _benchmark/result.md + echo "# Performance report!" > _benchmark/result.md + echo "## Rough benchmark" >> _benchmark/result.md cat _benchmark/rough-report.md >> _benchmark/result.md - echo "## Internal benchmark\n" >> _benchmark/result.md + echo "## Internal benchmark" >> _benchmark/result.md cat _benchmark/internal-report.md >> _benchmark/result.md - name: Find Performance report uses: peter-evans/find-comment@v3 diff --git a/scripts/perf-report/report-collector.js b/scripts/perf-report/report-collector.js index ca07bb8f3aa..5201c038880 100644 --- a/scripts/perf-report/report-collector.js +++ b/scripts/perf-report/report-collector.js @@ -24,8 +24,24 @@ export default new (class ReportCollector { if (process.env.CI) { return writeFile( fileURLToPath(new URL('../../_benchmark/internal-report.md', import.meta.url)), - this.#messageList.join('\n') + removeAnsiStyles(this.#messageList.join('\n')) ); } } })(); + +/** + * @param {string} text + * @returns {string} + */ +function removeAnsiStyles(text) { + const ansiRegex = new RegExp( + [ + '[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\\u0007)', + '(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))' + ].join('|'), + 'g' + ); + + return text.replace(ansiRegex, ''); +} From 0e0c0f60cbebb70e9f4e2f7e34613dc036e02e9b Mon Sep 17 00:00:00 2001 From: XiaoPi <530257315@qq.com> Date: Thu, 25 Apr 2024 20:25:54 +0800 Subject: [PATCH 4/4] Report added or removed tree-shaking stages --- scripts/perf-report/index.js | 63 +++++++++++++++++++++++++++++++----- 1 file changed, 55 insertions(+), 8 deletions(-) diff --git a/scripts/perf-report/index.js b/scripts/perf-report/index.js index e9b12ef0a9c..f06e723429a 100644 --- a/scripts/perf-report/index.js +++ b/scripts/perf-report/index.js @@ -173,9 +173,16 @@ function getSingleAverage(times, runs, discarded) { * @return {number} */ function printMeasurements(newAverage, previousAverage, filter = /.*/) { - const printedLabels = Object.keys(newAverage).filter(label => filter.test(label)); - console.info(''); - for (const label of printedLabels) { + const newPrintedLabels = Object.keys(newAverage).filter(predicateLabel); + const previousPrintedLabels = Object.keys(previousAverage).filter(predicateLabel); + + const newTreeShakings = newPrintedLabels.filter(isTreeShakingLabel); + const oldTreeShakings = previousPrintedLabels.filter(isTreeShakingLabel); + + const addedTreeShaking = newTreeShakings.length - oldTreeShakings.length; + let treeShakingCount = 0; + + for (const label of newPrintedLabels) { /** * @type {function(string): string} */ @@ -186,14 +193,54 @@ function printMeasurements(newAverage, previousAverage, filter = /.*/) { color = underline; } } - const text = `${label}: ${getFormattedTime( + const texts = []; + if (isTreeShakingLabel(label)) { + treeShakingCount++; + if (addedTreeShaking < 0 && treeShakingCount === newTreeShakings.length) { + texts.push(generateSingleReport(label)); + for (const label of oldTreeShakings.slice(addedTreeShaking)) { + const { time, memory } = previousAverage[label]; + texts.push(`${label}: ${time.toFixed(0)}ms, ${prettyBytes(memory)}, removed stage`); + } + } else if (addedTreeShaking > 0 && treeShakingCount > oldTreeShakings.length) { + texts.push(generateSingleReport(label, ', new stage')); + } else { + texts.push(generateSingleReport(label)); + } + } else { + texts.push(generateSingleReport(label)); + } + for (const text of texts) { + reportCollector.push(text); + console.info(color(text)); + } + } + return Math.max(newPrintedLabels.length, previousPrintedLabels.length) + 2; + + /** + * @param {string} label + */ + function predicateLabel(label) { + return filter.test(label); + } + + /** + * @param {string} label + * @param {string} addon + */ + function generateSingleReport(label, addon = '') { + return `${label}: ${getFormattedTime( newAverage[label].time, previousAverage[label]?.time - )}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}`; - reportCollector.push(text); - console.info(color(text)); + )}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}${addon}`; } - return printedLabels.length + 2; +} + +/** + * @param {string} label + */ +function isTreeShakingLabel(label) { + return label.startsWith('treeshaking pass'); } /**