test.yml

name: Tests CLI and SDK binaries.

defaults:
  run:
    shell: bash

on:
  workflow_call:
    inputs:
      platform:
        description: 'OS to run tests on, e.g.: ubuntu-latest'
        required: true
        type: string
      go-version:
        description: 'Version of the Go toolchain for the build'
        default: '1.17.x'
        required: false
        type: string
      python-version:
        description: 'Version of the Python toolchain for the build'
        default: '3.9.x'
        required: false
        type: string
      node-version:
        description: 'Version of the Node toolchain for the build'
        default: '14.x'
        required: false
        type: string
      dotnet-version:
        description: 'Version of the .NET toolchain for the build'
        default: '3.1.x'
        required: false
        type: string
      enable-coverage:
        description: 'Collects coverage stats; requires cov-enabled builds'
        default: false
        required: false
        type: boolean
    secrets:
      pulumi-access-token:
        required: true

env:
  GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  PULUMI_TEST_OWNER: "moolumi"
  PULUMI_ACCESS_TOKEN: ${{ secrets.pulumi-access-token }}
  PYTHON: python
  TESTPARALLELISM: 4
  DOTNET_CLI_TELEMETRY_OPTOUT: "true"

jobs:
  test:
    name: Test

    strategy:
      fail-fast: false
      matrix:
        test-suite:
          - run: |
              cd sdk/python
              make test_fast
              make test_auto
              make test_go

          - run: |
              cd sdk/dotnet
              make dotnet_test
              make test_auto
              make test_go

          - run: |
              cd sdk/nodejs
              make sxs_tests
              make unit_tests
              make test_auto
              make test_go
            test-parallelism: 2
            test-parallelism-windows: 1

          - run: |
              cd sdk/go
              make test_fast
              make test_auto

          - run: make test_pkg_nodejs
            require-build: true
            test-parallelism: 4
            test-parallelism-windows: 1

          - run: make test_pkg_python
            require-build: true
            test-parallelism: 4
            test-parallelism-windows: 1

          - run: make test_pkg_rest
            require-build: true
            test-parallelism: 2
            test-parallelism-windows: 1

          - run: make test_integration_dotnet
            require-build: true
            test-parallelism: 4
          - run: make test_integration_go
            require-build: true
            test-parallelism: 4
            test-parallelism-windows: 1
          - run: make test_integration_nodejs
            require-build: true
            test-parallelism: 4
            test-parallelism-windows: 1
          - run: make test_integration_python
            require-build: true
            test-parallelism: 4
            test-parallelism-windows: 1
          - run: make test_integration_rest
            require-build: true
            test-parallelism: 4
            test-parallelism-windows: 1
          - run: make test_integration_subpkgs
            require-build: true
            test-parallelism: 4
            test-parallelism-windows: 1

    env:
      PULUMI_HOME: ${{ github.workspace }}/home
      PULUMI_LOCAL_NUGET: ${{ github.workspace }}/nuget
      TEST_ALL_DEPS: ""

    runs-on: ${{ inputs.platform }}

    steps:
      - name: Checkout Repo
        uses: actions/checkout@v2
        with:
          ref: ${{ env.PR_COMMIT_SHA }}
      - name: Fetch Tags
        run: |
          git fetch --quiet --prune --unshallow --tags
      - name: Set test parallelism
        if: ${{ matrix.test-suite.test-parallelism }}
        run: |
          echo "TESTPARALLELISM=${{ matrix.test-suite.test-parallelism }}" >> $GITHUB_ENV
      - name: Set Windows test parallelism, temp dir
        if: ${{ matrix.test-suite.test-parallelism-windows }}
        run: |
          echo "TESTPARALLELISM=${{ matrix.test-suite.test-parallelism-windows }}" >> $GITHUB_ENV
          mkdir -p "${{ runner.temp }}/tmp"
      - name: Enable code coverage
        if: ${{ inputs.enable-coverage && (inputs.platform != 'windows-latest') }}
        run: |
          echo "PULUMI_TEST_COVERAGE_PATH=$(pwd)/coverage" >> $GITHUB_ENV
      # See: https://github.com/actions/virtual-environments/issues/2642#issuecomment-774988591
      - name: Configure Windows pagefile
        uses: aaronfriel/action-configure-pagefile@v2.0-beta.1
        if: ${{ runner.os == 'Windows' }}
        with:
            minimum-size: 4GB
            maximum-size: 4GB
            disk-root: 'D:'
      - name: Set up Go ${{ inputs.go-version }}
        uses: actions/setup-go@v2
        with:
          go-version: ${{ inputs.go-version }}
          check-latest: true
      - id: go-cache-paths
        run: |
          echo "::set-output name=go-build::$(go env GOCACHE)"
          echo "::set-output name=go-mod::$(go env GOMODCACHE)"
      - name: Go Cache
        uses: actions/cache@v2
        id: go-cache
        if: ${{ runner.os != 'Windows' }} # Note [Windows Go Cache] in build.yml
        with:
          path: |
              ${{ steps.go-cache-paths.outputs.go-build }}
              ${{ steps.go-cache-paths.outputs.go-mod }}
          key: ${{ runner.os }}-go-cache-${{ hashFiles('*/go.sum') }}
          restore-keys: |
            ${{ runner.os }}-go-
      - name: Set up Python ${{ inputs.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ inputs.python-version }}
          cache: pip
          cache-dependency-path: sdk/python/requirements.txt
      - name: Set up DotNet ${{ inputs.dotnet-version }}
        uses: actions/setup-dotnet@v1
        with:
          dotnet-version: ${{ inputs.dotnet-version }}
      - name: Set up Node ${{ inputs.node-version }}
        uses: actions/setup-node@v2
        with:
          node-version: ${{ inputs.node-version }}
          cache: yarn
          cache-dependency-path: sdk/nodejs/package.json
      - name: Uninstall pre-installed Pulumi (windows)
        if: ${{ inputs.platform == 'windows-latest' }}
        run: |
          if which pulumi.exe; then
            echo "Deleting pulumi"
            rm -rf "$(which pulumi.exe)/../pulumi*"
          fi
      - name: Install yarn
        run: |
          npm install -g yarn
      - name: Install Python deps
        run: |
          python -m pip install --upgrade pip requests wheel urllib3 chardet
      - name: Setup git
        run: |
          git config --global user.email "you@example.com"
          git config --global user.name "Your Name"
      - name: Set Go Dep path
        run: |
          echo "PULUMI_GO_DEP_ROOT=$(dirname $(pwd))" >> $GITHUB_ENV
      - name: Install pulumictl
        uses: jaxxstorm/action-install-gh-release@v1.7.1
        with:
          repo: pulumi/pulumictl
          tag: v0.0.31
          cache: enable
      - name: Install gotestsum
        uses: jaxxstorm/action-install-gh-release@v1.7.1
        with:
          repo: gotestyourself/gotestsum
          tag: v1.8.1
          cache: enable
      - name: Install goteststats
        uses: jaxxstorm/action-install-gh-release@v1.7.1
        with:
          repo: t0yv0/goteststats
          tag: v0.0.7
          cache: enable
      - name: Download pulumi-${{ runner.os }}-${{ runner.arch }}
        uses: actions/download-artifact@v2
        with:
          name: pulumi-${{ runner.os }}-${{ runner.arch }}
          path: artifacts/go
      - name: Install pulumi-language-python-exec
        run: |
          mkdir -p bin
          cp sdk/python/cmd/pulumi-language-python-exec bin/
      - name: Install Pulumi Go Binaries
        run: |
          echo "Checking contents of artifacts/go"
          find artifacts/go
          mkdir -p bin
          mv artifacts/go/pulumi*/* $PWD/bin/
          chmod a+x $PWD/bin/*
          echo "Checking contents of $PWD/bin"
          find $PWD/bin
      - name: Add ./bin to PATH
        run: |
          LOCAL_PATH=$(./scripts/normpath "${{ github.workspace }}/bin")
          echo "Adding LOCAL_PATH=$LOCAL_PATH to PATH"
          echo $LOCAL_PATH >> $GITHUB_PATH
      - name: Verify Pulumi Version
        run: |
          which pulumi   || echo "no pulumi"
          pulumi version || echo "no pulumi"
      - name: Download Pulumi .NET NuGet packages
        uses: actions/download-artifact@v2
        with:
          name: pulumi-nuget-packages
          # path set to match PULUMI_LOCAL_NUGET
          path: ${{ github.workspace }}/nuget
      - name: Add NuGet packages as a local NuGet source
        run: |
          echo $(which dotnet)
          echo $(dotnet --version)
          echo "PULUMI_LOCAL_NUGET=$PULUMI_LOCAL_NUGET"
          SOURCE=$(./scripts/normpath $PULUMI_LOCAL_NUGET)
          echo "SOURCE=$SOURCE"
          dotnet nuget add source $SOURCE
        env:
          PULUMI_LOCAL_NUGET: ${{ github.workspace }}/nuget
      - name: Inspect downloaded artifacts
        run: |
          find artifacts
          echo "PULUMI_LOCAL_NUGET=$PULUMI_LOCAL_NUGET"
          ls $PULUMI_LOCAL_NUGET
        env:
          PULUMI_LOCAL_NUGET: ${{ github.workspace }}/nuget
      - name: Ensure dependencies for the Node SDK
        run: |
          cd sdk/nodejs
          make ensure
      - name: Build the Node SDK
        run: |
          cd sdk/nodejs
          make build_package
          cd bin
          yarn link
      - name: Ensure dependencies for the Python SDK
        run: |
           cd sdk/python
           make ensure
      - name: Install Python SDK
        run: |
           cd sdk/python
           make build_package
      - name: build tests
        if: ${{ matrix.test-suite.require-build }}
        run: make test_build
        env:
          PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
          PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
      - name: run tests
        run: ${{ matrix.test-suite.run }}
        env:
          PULUMI_NODE_MODULES: ${{ runner.temp }}/opt/pulumi/node_modules
          PULUMI_ROOT: ${{ runner.temp }}/opt/pulumi
      - name: Convert Node coverage data
        if: ${{ inputs.platform != 'windows-latest' }}
        run: |
          cd sdk/nodejs
          if [ -e .nyc_output ]; then yarn run nyc report -r cobertura --report-dir $PULUMI_TEST_COVERAGE_PATH; fi
      - name: Merge Go coverage data
        if: ${{ inputs.enable-coverage && (inputs.platform != 'windows-latest') }}
        run: |
          pulumictl cover merge --in ./coverage --out ./coverage/go-all.txt
          rm ./coverage/*.cov || true
      - name: Upload code coverage
        if: ${{ inputs.enable-coverage && (inputs.platform != 'windows-latest') }}
        uses: codecov/codecov-action@v3
        with:
          directory: coverage
          files: '*'
          fail_ci_if_error: false
          verbose: true
      - name: Summarize Test Time by Package
        run: |
          mkdir -p test-results
          touch test-results/empty.json # otherwise goteststats fails below when no files match
          goteststats -statistic pkg-time test-results/*.json
      - name: Summarize Test Times by Individual Test
        run: |
          goteststats -statistic test-time test-results/*.json | head -n 100 || \
                      if [[ $? -eq 141 ]]; then true; else exit $?; fi