microbenchmarks.sh

#!/usr/bin/env bash

# Copyright 2019 The Knative Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

# -count argument when running go test
readonly BENCH_RUN_COUNT=${BENCH_RUN_COUNT:-5}

# -benchtime argument when running go test
readonly BENCH_RUN_TIME=${BENCH_RUN_TIME:-5s}

function microbenchmarks_run() {
  if [ "$1" != "" ]; then
    OUTPUT_FILE="$1"
  else
    OUTPUT_FILE="${ARTIFACTS:-$(mktemp -d)}/bench-result.txt"
  fi

  # Run all microbenchmarks
  go clean
  go test -bench=. -benchtime=$BENCH_RUN_TIME -count=$BENCH_RUN_COUNT -benchmem -run="^$" -v ./...   >> "$OUTPUT_FILE" || exit
}

# To run microbenchmarks on your machine and compare your revision with upstream/main:
#
# git fetch upstream
# source microbenchmarks.sh
# microbenchmarks_run_and_compare upstream/main
#
# NOTE: Hypothetically we should run these microbenchmarks on a machine running only a kernel and a shell,
# but this might be sometimes hard to achieve unless you have a spare computer to play with.
# When running microbenchmarks on your dev machine, make sure you have the bare minimum applications running on your laptop,
# otherwise you'll get a variance too high.
# Close Slack, the browser, the IDE, stop Docker, stop Netflix and grab a coffee while waiting for the results.
# Depending on the benchmark complexity/reproducibility, if the variance is too high, repeat the benchmarks.
function microbenchmarks_run_and_compare() {
  if [ "$1" == "" ] || [ $# -gt 1 ]; then
    echo "Error: Expecting an argument" >&2
    echo "usage: $(basename $0) revision_to_compare" >&2
    exit 1
  fi

  if [ "$2" == "" ]; then
    OUTPUT_DIR=${ARTIFACTS:-$(mktemp -d)}
  else
    OUTPUT_DIR="$2"
  fi

  mkdir -p "$OUTPUT_DIR"

  # Benchstat is required to compare the bench results
  GO111MODULE=off go get golang.org/x/perf/cmd/benchstat

  # Revision to use to compare with actual
  REVISION="$1"

  echo "--- Outputs will be at $OUTPUT_DIR"

  # Run this revision benchmarks
  microbenchmarks_run "$OUTPUT_DIR/new.txt"

  echo "--- This revision results:"
  cat "$OUTPUT_DIR/new.txt"

  # Run other revision benchmarks
  git checkout "$REVISION"
  microbenchmarks_run "$OUTPUT_DIR/old.txt"

  echo "--- $REVISION results:"
  cat "$OUTPUT_DIR/old.txt"

  # Print results in console
  echo "--- Benchstat:"
  benchstat "$OUTPUT_DIR/old.txt" "$OUTPUT_DIR/new.txt"

  # Generate html results
  benchstat -html "$OUTPUT_DIR/old.txt" "$OUTPUT_DIR/new.txt" >> "$OUTPUT_DIR/results.html"
}