bench-turborepo.yml

name: Benchmark Turborepo

on:
  workflow_dispatch:
  workflow_run:
    # Make sure this matches the name of the workflow in ./github/workflows/turborepo-release.yml.
    workflows: [Turborepo Release]
    types:
      - completed
  push:
    branches:
      - main
    paths:
      - "cli/**"
      - crates/turborepo*/**
      # - "benchmark/**" (we don't need to run benchmarks when the benchmark changes, next push will capture it)

jobs:
  benchmark:
    name: Benchmark turbo run
    timeout-minutes: 60
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3
      - uses: ./.github/actions/setup-node

      - name: Setup Turborepo Environment
        uses: ./.github/actions/setup-turborepo-environment
        with:
          github-token: "${{ secrets.GITHUB_TOKEN }}"

      - name: Build
        run: pnpm -F cli build:release

      - name: Run benchmarks
        run: pnpm -F @turbo/benchmark benchmark

      - name: Save to Tinybird
        run: |
          curl \
            -i \
            -F "ndjson=@./packages/turbo-benchmark/tinybird.ndjson" \
            -X POST \
            -H 'Authorization: Bearer ${{ secrets.TINYBIRD_TOKEN }}' \
            'https://api.us-east.tinybird.co/v0/datasources?format=ndjson&name=turbo_benchmarks&mode=append'

  time-to-first-task:
    name: Benchmark TTFT
    timeout-minutes: 60
    runs-on: ${{ matrix.os.runner }}
    strategy:
      fail-fast: false
      matrix:
        os:
          - name: ubuntu
            runner: ubuntu-latest
          - name: macos
            runner: macos-latest
          - name: windows
            runner: windows-latest

    steps:
      - uses: actions/checkout@v3
      - name: Set filename for profile
        id: filename
        shell: bash
        run: |
          echo 'file_basename=${{ matrix.os.name }}' >> "$GITHUB_OUTPUT"
          echo 'filename=${{ matrix.os.name }}.json' >> "$GITHUB_OUTPUT"

      - name: Display filename
        shell: bash
        run: echo "${{ steps.filename.outputs.filename }}"

      # setup Rust on windows, because apparently setup-turborepo-environment doesn't do this
      # TODO: put this logic in ./github/actions/setup-rust and use that instead
      - name: Add rustup for windows
        if: ${{ matrix.os.runner == 'windows-latest'}}
        run: rustup target add x86_64-pc-windows-gnu

      - name: Setup Turborepo Environment
        uses: ./.github/actions/setup-turborepo-environment
        with:
          windows: ${{ matrix.os.name == 'windows' }}
          github-token: "${{ secrets.GITHUB_TOKEN }}"

      - name: Build Turborepo from source
        run: pnpm -F cli build:release

      - name: Run benchmarks
        shell: bash
        # ttft script will normalize filepath and place the profile in the benchmark directory.
        run: pnpm -F @turbo/benchmark ttft "${{ steps.filename.outputs.filename }}"

      - name: Upload Artifacts
        uses: actions/upload-artifact@v3
        with:
          name: profiles # This name will be the folder each file gets downloaded to
          if-no-files-found: error
          # cwd is root of the repository, so we need the benchmark/ prefixed path
          path: |
            packages/turbo-benchmark/profiles/${{ steps.filename.outputs.filename }}
            packages/turbo-benchmark/profiles/${{ steps.filename.outputs.file_basename }}-ttft.json

  # Send each of the profiles generated from the previous job to TinyBird
  # We'll wait for all profiles to complete before sending.
  send-to-tinybird:
    name: Send to Tinybird
    needs: [time-to-first-task]
    runs-on: ubuntu-latest
    env:
      TINYBIRD_TOKEN: ${{secrets.TINYBIRD_TOKEN}}

    steps:
      - uses: actions/checkout@v3

      - name: Setup Node
        uses: ./.github/actions/setup-node

      - name: Download profiles
        uses: actions/download-artifact@v3
        with:
          name: profiles
          path: packages/turbo-benchmark/profiles/

      - name: Display TTFT Data
        shell: bash
        run: |
          ls -al packages/turbo-benchmark/profiles
          cat packages/turbo-benchmark/profiles/ubuntu-ttft.json
          cat packages/turbo-benchmark/profiles/macos-ttft.json
          cat packages/turbo-benchmark/profiles/windows-ttft.json

      - name: Send data to TinyBird
        shell: bash
        run: |
          cd packages/turbo-benchmark
          node -r esbuild-register ./src/ttft/tinybird.ts profiles/ubuntu-ttft.json ${{github.run_id}}
          node -r esbuild-register ./src/ttft/tinybird.ts profiles/macos-ttft.json ${{github.run_id}}
          node -r esbuild-register ./src/ttft/tinybird.ts profiles/windows-ttft.json ${{github.run_id}}

  send-to-slack:
    name: Send to Slack
    # Wait for send-to-tinybird so we can get aggregate data points
    # before sending to slack.
    needs: [send-to-tinybird]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Setup Node
        uses: ./.github/actions/setup-node

      - name: Download profiles
        uses: actions/download-artifact@v3
        with:
          name: profiles
          path: packages/turbo-benchmark/profiles/

      - name: Display TTFT Data
        shell: bash
        run: |
          ls -al packages/turbo-benchmark/profiles
          cat packages/turbo-benchmark/profiles/ubuntu-ttft.json
          cat packages/turbo-benchmark/profiles/macos-ttft.json
          cat packages/turbo-benchmark/profiles/windows-ttft.json

      # TODO: compare results to previous data and only post regressions
      - name: Create Slack payload
        shell: bash
        env:
          BLOB_READ_WRITE_TOKEN: ${{ secrets.BLOB_READ_WRITE_TOKEN }}
          TINYBIRD_TOKEN: ${{ secrets.TINYBIRD_TOKEN }}
        run: |
          cd packages/turbo-benchmark
          node -r esbuild-register ./src/ttft/slack.ts ${{github.run_id}}

      - name: Debug Slack payload
        shell: bash
        run: cat packages/turbo-benchmark/slack-payload.json | jq

      - name: Send payload to slack
        uses: slackapi/slack-github-action@v1.23.0
        with:
          payload-file-path: "packages/turbo-benchmark/slack-payload.json"
        env:
          SLACK_WEBHOOK_URL: "${{ secrets.TURBOREPO_PERF_BENCHMARK_SLACK_WEBHOOK_URL }}"