Report performance in CI

.github/workflows/performance-report.yml

@@ -0,0 +1,151 @@
+name: Performance Report
+env:
+  BUILD_BOOTSTRAP_CJS: mv dist dist-build && node dist-build/bin/rollup --config rollup.config.ts --configPlugin typescript --configTest --forceExit && rm -rf dist-build
+
+on:
+  pull_request:
+    types:
+      - synchronize
+      - opened
+      - reopened
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  build-artefacts:
+    strategy:
+      matrix:
+        settings:
+          - name: current
+            ref: refs/pull/${{ github.event.number }}/merge
+          - name: previous
+            ref: ${{github.event.pull_request.base.ref}}
+    name: Build ${{matrix.settings.name}} artefact
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout Commit
+        uses: actions/checkout@v4
+        with:
+          ref: ${{matrix.settings.ref}}
+      - name: Install Toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: nightly-2023-10-05
+          targets: x86_64-unknown-linux-gnu
+      - name: Cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            .cargo-cache
+            rust/target/
+          key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }}
+          restore-keys: cargo-cache
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: Cache Node Modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        with:
+          path: node_modules
+          key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
+      - name: Install dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts
+      - name: Build artefacts 123
+        run: npm exec -- concurrently -c green,blue 'npm:build:napi -- --release' 'npm:build:cjs' && npm run build:copy-native && ${{env.BUILD_BOOTSTRAP_CJS}} && npm run build:copy-native
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.settings.name }}
+          path: dist/
+          if-no-files-found: error
+
+  report:
+    needs: build-artefacts
+    permissions:
+      pull-requests: write
+    runs-on: ubuntu-latest
+    name: Report Performance
+    steps:
+      - name: Checkout Commit
+        uses: actions/checkout@v4
+        with:
+          ref: refs/pull/${{ github.event.number }}/merge
+      - name: Install Toolchain
+        uses: dtolnay/rust-toolchain@stable
+        with:
+          toolchain: nightly-2023-10-05
+          targets: x86_64-unknown-linux-gnu
+      - name: Cache cargo
+        uses: actions/cache@v4
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+            .cargo-cache
+            rust/target/
+          key: cargo-cache-${{ hashFiles('rust/Cargo.lock') }}
+          restore-keys: cargo-cache
+      - name: Setup Node
+        uses: actions/setup-node@v4
+        with:
+          node-version: 18
+      - name: Cache Node Modules
+        id: cache-node-modules
+        uses: actions/cache@v4
+        with:
+          path: node_modules
+          key: node-modules-${{ runner.os }}-${{ hashFiles('package-lock.json') }}
+      - name: Install dependencies
+        if: steps.cache-node-modules.outputs.cache-hit != 'true'
+        run: npm ci --ignore-scripts
+      - name: Download all artifacts
+        uses: actions/download-artifact@v4
+        with:
+          path: _benchmark
+      - name: Change rollup import in internal benchmark
+        run: |
+          echo "export { rollup as previousRollup, VERSION as previousVersion } from '../../_benchmark/previous/rollup.js';" > ./scripts/perf-report/rollup-artefacts.js
+          echo "export { rollup as newRollup } from '../../_benchmark/current/rollup.js';" >> ./scripts/perf-report/rollup-artefacts.js
+      - name: Run internal benchmark
+        run: node --expose-gc scripts/perf-report/index.js
+      - name: Install benchmark tool
+        run: cargo install --locked hyperfine
+      - name: Run Rough benchmark
+        run: |
+          hyperfine --warmup 1 --export-markdown _benchmark/rough-report.md --show-output --runs 3 \
+            'node _benchmark/previous/bin/rollup -i ./perf/entry.js -o _benchmark/result/previous.js' \
+            'node _benchmark/current/bin/rollup -i ./perf/entry.js -o _benchmark/result/current.js'
+      - name: Combine bechmark reports
+        run: |
+          echo "# Performance report!" > _benchmark/result.md
+          echo "## Rough benchmark" >> _benchmark/result.md
+          cat _benchmark/rough-report.md >> _benchmark/result.md
+          echo "## Internal benchmark" >> _benchmark/result.md
+          cat _benchmark/internal-report.md >> _benchmark/result.md
+      - name: Find Performance report
+        uses: peter-evans/find-comment@v3
+        id: findPerformanceReport
+        with:
+          issue-number: ${{ github.event.number }}
+          comment-author: 'github-actions[bot]'
+          body-includes: 'Performance report'
+      - name: Create or update Performance report
+        uses: peter-evans/create-or-update-comment@v4
+        id: createOrUpdatePerformanceReport
+        with:
+          comment-id: ${{ steps.findPerformanceReport.outputs.comment-id }}
+          issue-number: ${{ github.event.number }}
+          edit-mode: replace
+          body-path: _benchmark/result.md

package.json

@@ -66,7 +66,7 @@
     "lint:markdown:nofix": "prettier --check \"**/*.md\"",
     "lint:rust": "cd rust && cargo fmt && cargo clippy --fix --allow-dirty",
     "lint:rust:nofix": "cd rust && cargo fmt --check && cargo clippy",
-    "perf": "npm run build && node --expose-gc scripts/perf.js",
+    "perf": "npm run build && node --expose-gc scripts/perf-report/index.js",
     "prepare": "husky && node scripts/check-release.js || npm run build:prepare",
     "prepublishOnly": "node scripts/check-release.js && node scripts/prepublish.js",
     "postpublish": "node scripts/postpublish.js",

scripts/perf.js → scripts/perf-report/index.js

@@ -5,10 +5,9 @@ import { chdir } from 'node:process';
 import { fileURLToPath } from 'node:url';
 import { createColors } from 'colorette';
 import prettyBytes from 'pretty-bytes';
-import { rollup as previousRollup, VERSION as previousVersion } from 'rollup';
-// eslint-disable-next-line import/no-unresolved
-import { rollup as newRollup } from '../dist/rollup.js';
-import { runWithEcho } from './helpers.js';
+import { runWithEcho } from '../helpers.js';
+import reportCollector from './report-collector.js';
+import { newRollup, previousRollup, previousVersion } from './rollup-artefacts.js';
 
 /**
  * @typedef {Record<string,{memory:number,time:number}>} CollectedTimings
@@ -18,7 +17,7 @@ import { runWithEcho } from './helpers.js';
  * @typedef {Record<string, [number, number, number][]>} AccumulatedTimings
  */
 
-const PERF_DIRECTORY = new URL('../perf/', import.meta.url);
+const PERF_DIRECTORY = new URL('../../perf/', import.meta.url);
 const ENTRY = new URL('entry.js', PERF_DIRECTORY);
 const THREEJS_COPIES = 10;
 const { bold, underline, cyan, red, green } = createColors();
@@ -91,10 +90,12 @@ async function calculatePrintAndPersistTimings() {
 		);
 		clearLines(numberOfLinesToClear);
 	}
+	reportCollector.startRecord();
 	printMeasurements(
 		getAverage(accumulatedNewTimings, RUNS_TO_AVERAGE),
 		getAverage(accumulatedPreviousTimings, RUNS_TO_AVERAGE)
 	);
+	await reportCollector.outputMsg();
 }
 
 /**
@@ -172,9 +173,16 @@ function getSingleAverage(times, runs, discarded) {
  * @return {number}
  */
 function printMeasurements(newAverage, previousAverage, filter = /.*/) {
-	const printedLabels = Object.keys(newAverage).filter(label => filter.test(label));
-	console.info('');
-	for (const label of printedLabels) {
+	const newPrintedLabels = Object.keys(newAverage).filter(predicateLabel);
+	const previousPrintedLabels = Object.keys(previousAverage).filter(predicateLabel);
+
+	const newTreeShakings = newPrintedLabels.filter(isTreeShakingLabel);
+	const oldTreeShakings = previousPrintedLabels.filter(isTreeShakingLabel);
+
+	const addedTreeShaking = newTreeShakings.length - oldTreeShakings.length;
+	let treeShakingCount = 0;
+
+	for (const label of newPrintedLabels) {
 		/**
 		 * @type {function(string): string}
 		 */
@@ -185,16 +193,54 @@ function printMeasurements(newAverage, previousAverage, filter = /.*/) {
 				color = underline;
 			}
 		}
-		console.info(
-			color(
-				`${label}: ${getFormattedTime(
-					newAverage[label].time,
-					previousAverage[label]?.time
-				)}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}`
-			)
-		);
+		const texts = [];
+		if (isTreeShakingLabel(label)) {
+			treeShakingCount++;
+			if (addedTreeShaking < 0 && treeShakingCount === newTreeShakings.length) {
+				texts.push(generateSingleReport(label));
+				for (const label of oldTreeShakings.slice(addedTreeShaking)) {
+					const { time, memory } = previousAverage[label];
+					texts.push(`${label}: ${time.toFixed(0)}ms, ${prettyBytes(memory)}, removed stage`);
+				}
+			} else if (addedTreeShaking > 0 && treeShakingCount > oldTreeShakings.length) {
+				texts.push(generateSingleReport(label, ', new stage'));
+			} else {
+				texts.push(generateSingleReport(label));
+			}
+		} else {
+			texts.push(generateSingleReport(label));
+		}
+		for (const text of texts) {
+			reportCollector.push(text);
+			console.info(color(text));
+		}
+	}
+	return Math.max(newPrintedLabels.length, previousPrintedLabels.length) + 2;
+
+	/**
+	 * @param {string} label
+	 */
+	function predicateLabel(label) {
+		return filter.test(label);
+	}
+
+	/**
+	 * @param {string} label
+	 * @param {string} addon
+	 */
+	function generateSingleReport(label, addon = '') {
+		return `${label}: ${getFormattedTime(
+			newAverage[label].time,
+			previousAverage[label]?.time
+		)}, ${getFormattedMemory(newAverage[label].memory, previousAverage[label]?.memory)}${addon}`;
 	}
-	return printedLabels.length + 2;
+}
+
+/**
+ * @param {string} label
+ */
+function isTreeShakingLabel(label) {
+	return label.startsWith('treeshaking pass');
 }
 
 /**

scripts/perf-report/report-collector.js

@@ -0,0 +1,47 @@
+import { writeFile } from 'node:fs/promises';
+import { fileURLToPath } from 'node:url';
+
+export default new (class ReportCollector {
+	/**
+	 * @type {string[]}
+	 */
+	#messageList = [];
+	#isRecording = false;
+	startRecord() {
+		this.#isRecording = true;
+	}
+	/**
+	 * @param {string} message
+	 */
+	push(message) {
+		if (!this.#isRecording) return;
+		if (message.startsWith('#')) {
+			message = '##' + message;
+		}
+		this.#messageList.push(message);
+	}
+	outputMsg() {
+		if (process.env.CI) {
+			return writeFile(
+				fileURLToPath(new URL('../../_benchmark/internal-report.md', import.meta.url)),
+				removeAnsiStyles(this.#messageList.join('\n'))
+			);
+		}
+	}
+})();
+
+/**
+ * @param {string} text
+ * @returns {string}
+ */
+function removeAnsiStyles(text) {
+	const ansiRegex = new RegExp(
+		[
+			'[\\u001B\\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\\u0007)',
+			'(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))'
+		].join('|'),
+		'g'
+	);
+
+	return text.replace(ansiRegex, '');
+}

scripts/perf-report/rollup-artefacts.js

@@ -0,0 +1,3 @@
+export { rollup as previousRollup, VERSION as previousVersion } from 'rollup';
+// eslint-disable-next-line import/no-unresolved
+export { rollup as newRollup } from '../../dist/rollup.js';