Integrate the new benchmarks repo and update (#4310)

* Add benchmark git submodule * Add notes about using benchmarks repo to CONTRIBUTING * Update benchmarks commit * Attempt to fix up workflow files * Update benchmarks * Fix benchmark path in workflow * More fixes to workflow definitions * Update benchmarks commit * Workflow fixes * Fix log artifact name * Update PR benchmarks * Remove debugging steps from run-bench.yml * Update workflows * Trigger fake benchmark run * Update benchmarks * Fix bench names in pr-reporter * Update benchmarks * Update benchmarks with back-compat fix * Update benchmarks * Remove testing code
preactjs · Mar 29, 2024 · 757746a · 757746a
1 parent d3d57db
commit 757746a
Show file tree

Hide file tree

Showing 7 changed files with 124 additions and 41 deletions.
diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml
@@ -6,13 +6,13 @@ on:
     branches:
       - '**'
     paths:
-      - 'src/**.js'
+      - '**/src/**.js'
   push:
     branches:
       - main
-      - restructure
+      - v11
     paths:
-      - 'src/**.js'
+      - '**/src/**.js'
 
 jobs:
   build_test:
@@ -53,57 +53,57 @@ jobs:
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: todo
+      benchmark: todo/todo
       timeout: 10
 
   bench_text_update:
-    name: Bench text_update
+    name: Bench text-update
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: text_update
+      benchmark: text-update/text-update
       timeout: 10
 
   bench_many_updates:
-    name: Bench many_updates
+    name: Bench many-updates
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: many_updates
+      benchmark: many-updates/many-updates
       timeout: 10
 
-  bench_02_replace1k:
-    name: Bench 02_replace1k
+  bench_replace1k:
+    name: Bench replace1k
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: 02_replace1k
+      benchmark: table-app/replace1k
 
-  bench_03_update10th1k_x16:
+  bench_update10th1k:
     name: Bench 03_update10th1k_x16
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: 03_update10th1k_x16
+      benchmark: table-app/update10th1k
 
-  bench_07_create10k:
-    name: Bench 07_create10k
+  bench_create10k:
+    name: Bench create10k
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: 07_create10k
+      benchmark: table-app/create10k
 
   bench_hydrate1k:
     name: Bench hydrate1k
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: hydrate1k
+      benchmark: table-app/hydrate1k
 
   bench_filter_list:
-    name: Bench filter_list
+    name: Bench filter-list
     uses: ./.github/workflows/run-bench.yml
     needs: prepare
     with:
-      benchmark: filter_list
+      benchmark: filter-list/filter-list
       timeout: 10
diff --git a/.github/workflows/run-bench.yml b/.github/workflows/run-bench.yml
@@ -32,36 +32,90 @@ jobs:
     timeout-minutes: ${{ inputs.timeout }}
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
       - uses: actions/setup-node@v4
         with:
           node-version-file: 'package.json'
           cache: 'npm'
           cache-dependency-path: '**/package-lock.json'
+
+      # Setup pnpm
+      - name: Install pnpm
+        uses: pnpm/action-setup@v3
+        with:
+          version: 8
+          run_install: false
+      - name: Get pnpm store directory
+        id: pnpm-cache
+        run: |
+          echo "pnpm_cache_dir=$(pnpm store path)" >> $GITHUB_OUTPUT
+      - uses: actions/cache@v4
+        name: Setup pnpm cache
+        with:
+          path: ${{ steps.pnpm-cache.outputs.pnpm_cache_dir }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      # Install benchmark dependencies
       - uses: actions/download-artifact@v3
         with:
           name: bench-environment
+      - name: Move tarballs from env to correct location
+        run: |
+          ls -al
+          mv preact-local.tgz benchmarks/dependencies/preact/local-pinned/preact-local-pinned.tgz
+          ls -al benchmarks/dependencies/preact/local-pinned
+          mv preact-main.tgz benchmarks/dependencies/preact/main/preact-main.tgz
+          ls -al benchmarks/dependencies/preact/main
       - name: Install deps
-        working-directory: benches
-        run: npm ci
+        working-directory: benchmarks
+        # Set the CHROMEDRIVER_FILEPATH so the chromedriver npm package uses the
+        # correct binary when its installed
+        run: |
+          export CHROMEDRIVER_FILEPATH=$(which chromedriver)
+          pnpm install
+          # Install local dependencies with --no-frozen-lockfile to ensure local tarballs
+          # are installed regardless of if they match the integrity hash stored in the lockfile
+          pnpm install --no-frozen-lockfile --filter ./dependencies
+
+      # Run benchmark
       - name: Run benchmark
-        working-directory: benches
+        working-directory: benchmarks
         run: |
           export CHROMEDRIVER_FILEPATH=$(which chromedriver)
-          npm run bench ${{ inputs.benchmark }}.html -- --trace=${{ inputs.trace }}
+          pnpm run bench apps/${{ inputs.benchmark }}.html -d preact@local-pinned -d preact@main --trace=${{ inputs.trace }}
+
+      # Prepare output
+      - name: Anaylze logs if present
+        working-directory: benchmarks
+        run: '[ -d out/logs ] && pnpm run analyze ${{ inputs.benchmark }} || echo "No logs to analyze"'
+      - name: Tar logs if present
+        working-directory: benchmarks
+        run: |
+          if [ -d out/logs ]; then
+            LOGS_FILE=out/${{ inputs.benchmark }}_logs.tgz
+            mkdir -p $(dirname $LOGS_FILE)
+            tar -zcvf $LOGS_FILE out/logs
+          else
+            echo "No logs found"
+          fi
+
+      # Upload results and logs
+      - name: Calculate log artifact name
+        id: log-artifact-name
+        run: |
+          NAME=$(echo "${{ inputs.benchmark }}" | sed -r 's/[\/]+/_/g')
+          echo "artifact_name=logs_$NAME" >> $GITHUB_OUTPUT
       - name: Upload results
         uses: actions/upload-artifact@v3
         with:
           name: results
-          path: benches/results/${{ inputs.benchmark }}.json
-      - name: Anaylze logs if present
-        working-directory: benches
-        run: '[ -d logs ] && npm run analyze ${{ inputs.benchmark }} || echo "No logs to analyze"'
-      - name: Tar logs if present
-        working-directory: benches
-        run: '[ -d logs ] && tar -zcvf ${{ inputs.benchmark}}_logs.tgz logs || echo "No logs found"'
+          path: benchmarks/out/results/${{ inputs.benchmark }}.json
       - name: Upload logs
         uses: actions/upload-artifact@v3
         with:
-          name: ${{ inputs.benchmark}}_logs
-          path: benches/${{ inputs.benchmark}}_logs.tgz
+          name: ${{ steps.log-artifact-name.outputs.artifact_name }}
+          path: benchmarks/out/${{ inputs.benchmark }}_logs.tgz
           if-no-files-found: ignore
diff --git a/.github/workflows/single-bench.yml b/.github/workflows/single-bench.yml
@@ -7,14 +7,14 @@ on:
         description: 'Which benchmark to run'
         type: choice
         options:
-          - 02_replace1k
-          - 03_update10th1k_x16
-          - 07_create10k
-          - filter_list
-          - hydrate1k
-          - many_updates
-          - text_update
-          - todo
+          - table-app/replace1k
+          - table-app/update10th1k
+          - table-app/create10k
+          - table-app/hydrate1k
+          - filter_list/filter-list
+          - many-updates/many-updates
+          - text-update/text-update
+          - todo/todo
         required: true
       base:
         description: 'The branch name, tag, or commit sha of the version of preact to benchmark against.'

diff --git a/.gitmodules b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "benchmarks"]
+	path = benchmarks
+	url = https://github.com/preactjs/benchmarks
diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -131,6 +131,30 @@ it.only('should test something', () => {
 - Check the JSDoc block right above the function definition to understand what it does. It contains a short description of each function argument and what it does.
 - Check the callsites of a function to understand how it's used. Modern editors/IDEs allow you to quickly find those, or use the plain old search feature instead.
 
+## Benchmarks
+
+We have a benchmark suite that we use to measure the performance of Preact. Our benchmark suite lives in our [preactjs/benchmarks repository](https://github.com/preactjs/benchmarks), but is included here as Git submodule. To run the benchmarks, first ensure [PNPM](https://pnpm.io/installation) is installed on your system and initialize and setup the submodule (it uses `pnpm` as a package manager):
+
+```bash
+pnpm -v # Make sure pnpm is installed
+git submodule update --init --recursive
+cd benchmarks
+pnpm i
+```
+
+Then you can run the benchmarks:
+
+```bash
+# In the benchmarks folder
+pnpm run bench
+```
+
+Checkout the README in the benchmarks folder for more information on running benchmarks.
+
+> **Note:** When switching branches, git submodules are not automatically updated to the commit of the new branch - it stays at the commit of the previous branch. This can be a feature! It allows you to work in different branches with the latest versions of the benchmarks - especially if you have made changes to the benchmarks.
+>
+> However if you want to switch branches and also update the benchmarks to the latest commit of the new branch, you can run `git submodule update --recursive` after switching branches, or run `git checkout --recurse-submodules` when checking out a new branch.
+
 ## FAQ
 
 ### Why does the JSDoc use TypeScript syntax to specify types?

diff --git a/benchmarks b/benchmarks
diff --git a/package.json b/package.json
@@ -186,7 +186,8 @@
     "test/fixtures",
     "test/ts/",
     "*.ts",
-    "dist"
+    "dist",
+    "benchmarks"
   ],
   "prettier": {
     "singleQuote": true,