lint.yml

name: Lint

on:
  pull_request:
  push:
    branches:
      - master
      - main
      - release/*
      - landchecks/*
  workflow_dispatch:

jobs:
  lintrunner:
    runs-on: linux.20_04.16x
    steps:
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
          fetch-depth: 1

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/.github/requirements-gha-cache.txt

      - name: Install requirements
        run: |
          pip install -r .github/requirements-gha-cache.txt --user

      - name: Initialize lint dependencies
        run: lintrunner init

      - name: Do build steps necessary for linters
        run: |
          python3 -m tools.linter.clang_tidy.generate_build_files
          python3 -m tools.generate_torch_version --is_debug=false
          python3 -m tools.pyi.gen_pyi \
            --native-functions-path aten/src/ATen/native/native_functions.yaml \
            --tags-path aten/src/ATen/native/tags.yaml \
            --deprecated-functions-path "tools/autograd/deprecated.yaml"

      - name: Run lintrunner on all files
        run: |
          set +e
          if ! lintrunner --force-color --all-files --tee-json=lint.json; then
              echo ""
              echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`.\e[0m"
              echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m"
              exit 1
          fi

      - name: Store annotations
        if: always() && github.event_name == 'pull_request'
        # Don't show this as an error; the above step will have already failed.
        continue-on-error: true
        run: |
          # Use jq to massage the JSON lint output into GitHub Actions workflow commands.
          jq --raw-output \
            '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \
            lint.json

  quick-checks:
    name: quick-checks
    runs-on: linux.20_04.4x
    steps:
      # [see note: pytorch repo ref]
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
          fetch-depth: 1
      - name: Clean PyTorch checkout
        run: |
          # Remove any artifacts from the previous checkouts
          git clean -fxd
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.x
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/requirements.txt
      - name: Install requirements
        id: requirements
        run: pip install -r requirements.txt --user
      - name: Ensure no non-breaking spaces
        if: always()
        run: |
          # NB: We use 'printf' below rather than '\u000a' since bash pre-4.2
          # does not support the '\u000a' syntax (which is relevant for local linters)
          (! git --no-pager grep -In "$(printf '\xC2\xA0')" -- . || (echo "The above lines have non-breaking spaces (U+00A0); please convert them to spaces (U+0020)"; false))
      - name: Ensure cross-OS compatible file names
        if: always()
        run: |
          (! git ls-files | grep -E '([<>:"|?*]|[ .]$)' || (echo "The above file names are not valid across all operating systems. Please ensure they don't contain the characters '<>:""|?*' and don't end with a white space or a '.' "; false))
      - name: Ensure no versionless Python shebangs
        if: always()
        run: |
          (! git --no-pager grep -In '#!.*python$' -- . || (echo "The above lines have versionless Python shebangs; please specify either python2 or python3"; false))
      - name: C++ docs check
        if: ${{ always() && steps.requirements.outcome == 'success' }}
        run: |
          sudo apt-get install -y doxygen
          cd docs/cpp/source && ./check-doxygen.sh
      - name: CUDA kernel launch check
        if: ${{ always() && steps.requirements.outcome == 'success' }}
        run: |
          set -eux
          python torch/testing/_internal/check_kernel_launches.py |& tee "${GITHUB_WORKSPACE}"/cuda_kernel_launch_checks.txt

  pr-sanity-checks:
    name: pr-sanity-checks
    runs-on: linux.20_04.4x
    # Only run this on pull requests
    if: github.event_name == 'pull_request' && !contains(github.event.pull_request.labels.*.name, 'skip-pr-sanity-checks')
    steps:
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
          fetch-depth: -1
      - name: PR size check
        env:
          BASE: ${{ github.event.pull_request.base.sha }}
          HEAD: ${{ github.event.pull_request.head.sha }}
        run: |
          bash .github/scripts/pr-sanity-check.sh

  workflow-checks:
    name: workflow-checks
    runs-on: linux.20_04.4x
    steps:
      # [see note: pytorch repo ref]
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
          fetch-depth: 1
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.x
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/requirements.txt
            **/.github/requirements-gha-cache.txt
      - name: Install requirements
        id: requirements
        run: |
          pip install -r requirements.txt --user
      - name: Install Jinja2
        run: |
          pip install Jinja2==3.0.1 --user
      - name: Regenerate workflows
        id: generate_workflows
        run: .github/scripts/generate_ci_workflows.py
      - name: Assert that regenerating the workflows didn't change them
        run: |
          if ! .github/scripts/report_git_status.sh .github/workflows; then
            echo
            echo 'As shown by the above diff, the committed .github/workflows'
            echo 'are not up to date according to .github/templates.'
            echo 'Please run this command, commit, and push again to your PR:'
            echo
            echo '    .github/scripts/generate_ci_workflows.py'
            echo
            echo 'If running that command does nothing, you may need to rebase'
            echo 'onto a more recent commit from the PyTorch master branch.'
            false
          fi
      - name: Check that jobs will be cancelled
        if: ${{ always() && steps.generate_workflows.outcome == 'success' }}
        run: |
          .github/scripts/ensure_actions_will_cancel.py

  toc:
    name: toc
    runs-on: linux.20_04.4x
    # https://github.com/actions/virtual-environments/issues/599#issuecomment-602754687
    env:
      NPM_CONFIG_PREFIX: ~/.npm-global
    steps:
      # [see note: pytorch repo ref]
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
          fetch-depth: 1
      # This is not a node project so there is no package-lock.json to cache
      - name: Setup Node
        uses: actions/setup-node@v3
      - name: Install markdown-toc
        run: npm install -g markdown-toc
      - name: Regenerate ToCs and check that they didn't change
        run: |
          set -eu
          export PATH=~/.npm-global/bin:"$PATH"
          for FILE in $(git grep -Il '<!-- toc -->' -- '**.md'); do
            markdown-toc --bullets='-' -i "$FILE"
          done

          if ! .github/scripts/report_git_status.sh .; then
            echo
            echo 'As shown by the above diff, the table of contents in one or'
            echo 'more Markdown files is not up to date with the file contents.'
            echo 'You can either apply that Git diff directly to correct the'
            echo 'table of contents, or if you have npm installed, you can'
            echo 'install the npm package markdown-toc and run the following'
            # shellcheck disable=SC2016
            echo 'command (replacing $FILE with the filename for which you want'
            echo 'to regenerate the table of contents):'
            echo
            # shellcheck disable=SC2016
            echo "    markdown-toc --bullets='-' -i \"\$FILE\""
            false
          fi

  check-labels:
    name: Check labels
    runs-on: linux.20_04.16x
    if: github.event_name == 'pull_request'
    steps:
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
          fetch-depth: 1

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/.github/requirements-gha-cache.txt

      - name: Install requirements
        id: requirements
        run: |
          pip install -r .github/requirements-gha-cache.txt --user

      - name: Check labels
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PR_NUM: ${{ github.event.number }}
        run: |
          set -ex
          python3 .github/scripts/check_labels.py "${PR_NUM}"

  test-tools:
    name: Test tools
    if: ${{ github.repository == 'pytorch/pytorch' }}
    runs-on: linux.20_04.4x
    steps:
      # [see note: pytorch repo ref]
      # deep clone (fetch-depth 0) required, to allow us to use git log
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/requirements.txt
            **/requirements-flake8.txt
            **/.circleci/docker/requirements-ci.txt
            **/.github/requirements-gha-cache.txt
      - name: Install dependencies
        # mypy and boto3 versions copied from
        # .circleci/docker/common/install_conda.sh
        run: |
          set -eux
          pip install -r requirements.txt
          pip install boto3==1.19.12
          pip install typing-extensions==3.10 --user
          pip install -r requirements-flake8.txt --user
          pip install rockset==0.8.10 --user
          pip install -r requirements.txt --user
          pip install mypy==0.960 --user
          make setup_lint
      - name: Test tools
        run: |
          python3 -m unittest discover -vs tools/test -p 'test_*.py'
          python3 -m unittest discover -vs .github/scripts -p 'test_*.py'

  test_collect_env:
    if: ${{ github.repository == 'pytorch/pytorch' }}
    name: Test collect_env
    runs-on: linux.20_04.4x
    strategy:
      matrix:
        test_type: [with_torch, without_torch, older_python_version]
    steps:
      # [see note: pytorch repo ref]
      # deep clone (fetch-depth 0) required, to allow us to use git log
      - name: Checkout PyTorch
        uses: pytorch/pytorch/.github/actions/checkout-pytorch@master
        with:
          submodules: false
          fetch-depth: 1
      - name: Setup Python 3.5
        if: matrix.test_type == 'older_python_version'
        uses: actions/setup-python@v4
        with:
          python-version: 3.5
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/requirements.txt
      - name: Setup Python 3.8
        if: matrix.test_type != 'older_python_version'
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
          architecture: x64
          check-latest: false
          cache: pip
          cache-dependency-path: |
            **/requirements.txt
      - name: Install torch
        if: matrix.test_type == 'with_torch'
        run: |
          pip install -r requirements.txt
          # Doesn't really matter what torch version, we just need ANY torch installed
          pip install 'torch==1.*'
      - name: Run collect_env.py
        run: |
          # All we need to see is that it passes
          python3 torch/utils/collect_env.py

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true