performance-benchmarks-bayesmark.yml

name: Performance benchmarks with bayesmark

on:
  workflow_dispatch:
    inputs:
      sampler-list:
        description: 'Sampler List: A list of samplers to check the performance. Should be a whitespace-separated list of Optuna samplers. Each sampler must exist under `optuna.samplers` or `optuna.integration`.'
        required: false
        default: 'TPESampler CmaEsSampler'
      sampler-kwargs-list:
        description: 'Sampler Arguments List: A list of sampler keyword arguments. Should be a whitespace-separated list of json format dictionaries.'
        required: false
        default: '{"multivariate":true,"constant_liar":true} {}'
      pruner-list:
        description: 'Pruner List: A list of pruners to check the performance. Should be a whitespace-separated list of Optuna pruners. Each pruner must exist under `optuna.pruners`.'
        required: false
        default: 'NopPruner'
      pruner-kwargs-list:
        description: 'Pruner Arguments List: A list of pruner keyword arguments. Should be a whitespace-separated list of json format dictionaries.'
        required: false
        default: '{}'
      budget:
        description: 'Number of Trials if the pruning is not enabled. If the pruning is enabled, the total number of steps is equal to `budget * (steps per trial)`.'
        required: false
        default: '80'
      n-runs:
        description: 'Number of Studies'
        required: false
        default: '10'
      plot-warmup:
        description: Include warm-up steps in plots.
        type: boolean
        default: true

jobs:
  benchmarks:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        dataset: [breast, digits, iris, wine, diabetes]
        model: [kNN, SVM, DT, RF, MLP-sgd, ada, linear]

    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: Setup Python3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Install Python libraries
      run: |
        python -m pip install --upgrade pip
        pip install --progress-bar off -U setuptools wheel

        pip install --progress-bar off .
        python -c 'import optuna'
        optuna --version

        pip install --progress-bar off .[benchmark] --extra-index-url https://download.pytorch.org/whl/cpu
        pip install --progress-bar off bayesmark matplotlib pandas

    - name: Output installed packages
      run: |
        pip freeze --all
    - name: Output dependency tree
      run: |
        pip install pipdeptree
        pipdeptree


    - name: Run performance benchmark
      run: |
        python benchmarks/run_bayesmark.py \
          --dataset ${{ matrix.dataset }} \
          --model ${{ matrix.model }} \
          --name-prefix "" \
          --budget ${{ github.event.inputs.budget }} \
          --n-runs ${{ github.event.inputs.n-runs }} \
          --sampler-list '${{ github.event.inputs.sampler-list }}' \
          --sampler-kwargs-list '${{ github.event.inputs.sampler-kwargs-list }}' \
          --pruner-list '${{ github.event.inputs.pruner-list }}' \
          --pruner-kwargs-list '${{ github.event.inputs.pruner-kwargs-list }}' \
          --plot-warmup '${{ github.event.inputs.plot-warmup }}'

    - name: Upload plot
      uses: actions/upload-artifact@v3
      with:
        name: benchmark-plots
        path: plots

    - name: Upload partial report
      uses: actions/upload-artifact@v3
      with:
        name: partial-reports
        path: partial

  report:
    needs: benchmarks
    runs-on: ubuntu-latest

    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: Download partial reports
      uses: actions/download-artifact@v2
      with:
        name: partial-reports
        path: partial

    - name: Setup Python3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Install Python libraries
      run: |
        python -m pip install --upgrade pip
        pip install --progress-bar off -U setuptools wheel
        pip install --progress-bar off numpy scipy pandas Jinja2

    - name: Output installed packages
      run: |
        pip freeze --all
    - name: Output dependency tree
      run: |
        pip install pipdeptree
        pipdeptree


    - name: Run benchmark report builder
      run: |
        python benchmarks/bayesmark/report_bayesmark.py

    - name: Upload report
      uses: actions/upload-artifact@v3
      with:
        name: benchmark-report
        path: report

    - name: Cleanup partial reports
      uses: geekyeggo/delete-artifact@v1
      with:
        name: partial-reports