Skip to content

Benchmark other Bundlers #603

Benchmark other Bundlers

Benchmark other Bundlers #603

name: Benchmark other Bundlers
on:
schedule:
# Run every week on Sunday 12:42
- cron: "42 12 * * 0"
workflow_dispatch:
env:
NODE_OPTIONS: "--max-old-space-size=32768"
jobs:
bench:
strategy:
fail-fast: false
matrix:
bench:
# One of Turbopack CSR or SSR with 1000 modules is benchmarked in every run
# to create a baseline result for normalization (should the runners performance vary between jobs)
# This measures Next.js canary
- name: Next canary turbo
title: Next.js canary Turbo
run_args: '"(Turbopack SSR/1000|Next.js canary Turbo)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Next.js 13
- name: next 13 turbo
title: Next.js 13 Turbo
run_args: '"(Turbopack SSR/1000|Next.js 13 Turbo)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Next.js canary
- name: next canary
title: Next.js canary
run_args: '"(Turbopack SSR/1000|Next.js canary webpack)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Next.js 14
- name: next 14
title: Next.js 14
run_args: '"(Turbopack SSR/1000|Next.js 14)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Next.js 13
- name: next 13
title: Next.js 13
run_args: '"(Turbopack SSR/1000|Next.js 13 webpack)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Next.js 12
- name: next 12
title: Next.js 12
run_args: '"(Turbopack SSR/1000|Next.js 12)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Vite
- name: vite
title: Vite
run_args: '"(Turbopack CSR/1000|Vite)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Parcel
- name: parcel
title: Parcel
run_args: '"(Turbopack CSR/1000|Parcel)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
# This measures Webpack
- name: webpack
title: Webpack
run_args: '"(Turbopack CSR/1000|Webpack)"'
TURBOPACK_BENCH_COUNTS: 100,500,1000
TURBOPACK_BENCH_BUNDLERS: all
runs-on: ubuntu-latest-8-core-oss
name: Bench - ${{ matrix.bench.title }}
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-node
- uses: ./.github/actions/setup-rust
with:
shared-cache-key: benchmark-bundlers
github-token: ${{ secrets.GITHUB_TOKEN }}
- name: Clear potentially cached benchmarks
run: rm -rf target/criterion
- name: Build benchmarks
timeout-minutes: 120
run: cargo bench --no-run -p turbopack-bench -p turbopack-cli
- name: Run cargo bench
timeout-minutes: 180
env:
TURBOPACK_BENCH_BUNDLERS: ${{ matrix.bench.TURBOPACK_BENCH_BUNDLERS }}
TURBOPACK_BENCH_COUNTS: ${{ matrix.bench.TURBOPACK_BENCH_COUNTS }}
run: cargo bench -p turbopack-bench -p turbopack-cli -- ${{ matrix.bench.run_args }}
- name: Install critcmp
if: always()
uses: baptiste0928/cargo-install@v1
with:
crate: critcmp
- name: Compare results
if: always()
run: critcmp --group "([^/]+/)[^/]+(?:/)(.+)" base
- name: Export results
if: always()
run: critcmp --export base > raw.json
- name: Upload results
if: always()
uses: actions/upload-artifact@v3
with:
name: ${{ matrix.bench.name }}
path: raw.json
commit_results:
name: Commit benchmark-data
needs: bench
if: always()
runs-on: ubuntu-latest
steps:
- name: Get current date
id: date
run: |
echo "year=$(date +'%Y')" >> $GITHUB_OUTPUT
echo "month=$(date +'%m')" >> $GITHUB_OUTPUT
echo "date=$(date +'%s')" >> $GITHUB_OUTPUT
echo "pretty=$(date +'%Y-%m-%d %H:%M')" >> $GITHUB_OUTPUT
- name: Checkout benchmark-data
uses: actions/checkout@v3
with:
ref: benchmark-data
- name: Download benchmark data
uses: actions/download-artifact@v3
with:
path: data/${{ steps.date.outputs.year }}/${{ steps.date.outputs.month }}/ubuntu-latest-8-core/${{ steps.date.outputs.date }}-${{ github.sha }}
- name: Git pull
run: git pull --depth=1 --no-tags origin benchmark-data
- name: Push data to branch
uses: stefanzweifel/git-auto-commit-action@v4
with:
commit_message: Benchmark result for ${{ steps.date.outputs.pretty }} (${{ github.sha }})