Merge pull request #176 from lsst-sqre/tickets/DM-29123

[DM-29123] Improve CI caching and Docker builds

.dockerignore

@@ -0,0 +1,152 @@
+# Some additional things to ignore beyond what .gitignore already handles.
+# We cannot exclude the .git directory because setuptools_scm requires it.
+docs/
+examples/
+tests/
+ui/node_modules/
+
+# Everything below this point is a copy of .gitignore.
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+pip-wheel-metadata/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+docs/api/
+docs/_static/*.png
+
+# PyBuilder
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Emacs temporary files
+.#*
+\#*#
+
+# Modified to set up the local development server.
+/examples/secrets/github-client-secret

.github/workflows/ci.yaml

@@ -1,28 +1,14 @@
 name: CI
 
-on: [push]
+"on": [push]
 
 jobs:
-  test:
+  ui:
     runs-on: ubuntu-latest
 
-    strategy:
-      matrix:
-        python:
-          - 3.8
-          - 3.9
-        database:
-          - SQLite
-          - PostgreSQL
-
     steps:
       - uses: actions/checkout@v2
 
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: ${{ matrix.python }}
-
       - name: Read .nvmrc
         id: node_version
         run: echo ::set-output name=NODE_VERSION::$(cat .nvmrc)
@@ -32,8 +18,21 @@ jobs:
         with:
           node-version: ${{ steps.node_version.outputs.NODE_VERSION }}
 
-      - name: Cache Node.js modules
+      # First try to restore the fully-installed node modules.  If that
+      # works (no changes to the JavaScript layer), skip npm i and
+      # restoring the cache of downloaded modules.  If that fails, restore
+      # the cache of the downloaded modules and then run npm
+      # clean-install.
+      - name: Cache installed Node modules
         uses: actions/cache@v2
+        id: node-cache
+        with:
+          path: ./ui/node_modules
+          key: node-${{ steps.node_version.outputs.NODE_VERSION }}-${{ hashFiles('**/package-lock.json') }}
+
+      - name: Cache downloaded Node.js modules
+        uses: actions/cache@v2
+        if: steps.node-cache.outputs.cache-hit != 'true'
         with:
           path: ~/.npm
           key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
@@ -42,25 +41,79 @@ jobs:
 
       - name: Install Node dependencies
         run: npm ci
+        if: steps.node-cache.outputs.cache-hit != 'true'
         working-directory: ./ui
 
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.8
+
+      # This has to happen after installing Node modules because we run
+      # eslint and it wants react to be already installed.  We therefore
+      # do all the linting here instead of during the test job.
+      - name: Run pre-commit
+        uses: pre-commit/action@v2.0.0
+
       - name: Build the UI
-        run: make ui
+        run: npm run build
+        working-directory: ./ui
+
+      # Cache the built web UI in a build artifact so that it can be used
+      # by both the test job and the docker job.  We only use this
+      # artifact internally in this workflow, so only keep it for a day,
+      # not the full 90 day default.
+      - name: Cache UI artifact
+        uses: actions/upload-artifact@v2
+        with:
+          name: ui
+          path: ui/public
+          retention-days: 1
+
+  test:
+    runs-on: ubuntu-latest
+    needs: [ui]
+
+    strategy:
+      matrix:
+        python:
+          - 3.8
+          - 3.9
+        database:
+          - SQLite
+          - PostgreSQL
+
+    steps:
+      - uses: actions/checkout@v2
+
+      # Reuse the built UI from the ui job.
+      - name: Restore UI artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: ui
+          path: ui/public
+
+      - name: Set up Python
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python }}
 
       - name: Install tox
-        run: pip install tox
+        run: pip install tox tox-docker
 
       - name: Cache tox environments
         id: cache-tox
         uses: actions/cache@v2
         with:
           path: .tox
-          # requirements/*.txt, pyproject.toml, and .pre-commit-config.yaml
-          # have versioning info that would impact the tox environment.
-          key: tox-${{ hashFiles('requirements/*.txt') }}-${{ hashFiles('pyproject.toml') }}-${{ hashFiles('.pre-commit-config.yaml') }}
+          # requirements/*.txt and tox.ini have versioning or dependency
+          # information that would impact the tox environment.
+          key: tox-${{ matrix.python }}-${{ hashFiles('requirements/*.txt') }}-${{ hashFiles('tox.ini') }}
+          restore-keys: |
+            tox-${{ matrix.python }}-${{ hashFiles('requirements/*.txt') }}-
 
       - name: Run tox (SQLite)
-        run: tox -e lint,py,coverage-report,typing
+        run: tox -e py,coverage-report,typing
         if: matrix.database == 'SQLite'
 
       - name: Run tox (PostgreSQL)
@@ -71,64 +124,57 @@ jobs:
     runs-on: ubuntu-latest
     needs: [test]
 
+    # Only do Docker builds of ticket branches and tagged releases.
+    if: startsWith(github.ref, 'refs/tags/') || startsWith(github.ref, 'refs/heads/tickets/')
+
     steps:
-      - uses: actions/checkout@v2.3.4
+      - uses: actions/checkout@v2
+
+      # Reuse the built UI from the ui job.
+      - name: Restore UI artifact
+        uses: actions/download-artifact@v2
+        with:
+          name: ui
+          path: ui/public
 
       - name: Define the Docker tag
         id: vars
-        run: echo ::set-output name=tag::$(echo ${GITHUB_REF} | sed -E 's,refs/(heads|tags)/,,' | sed -E 's,/,-,g')
+        run: echo ::set-output name=tag::$(scripts/docker-tag.sh "$GITHUB_REF")
 
       - name: Print the tag
         id: print
         run: echo ${{steps.vars.outputs.tag}}
 
-      - name: Log into Docker Hub
-        run: echo ${{ secrets.DOCKER_TOKEN }} | docker login --username ${{ secrets.DOCKER_USERNAME }} --password-stdin
-
-      - name: Pull previous images
-        run: |
-          docker pull lsstsqre/gafaelfawr:deps-${{steps.vars.outputs.tag}} || true
-          docker pull lsstsqre/gafaelfawr:${{steps.vars.outputs.tag}} || true
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
 
-      - name: Build the dependencies Docker image
-        run: |
-          docker build --target dependencies-image \
-            --cache-from=lsstsqre/gafaelfawr:deps-${{steps.vars.outputs.tag}} \
-            --tag lsstsqre/gafaelfawr:deps-${{steps.vars.outputs.tag}} .
-
-      - name: Build the runtime Docker image
-        run: |
-          docker build --target runtime-image \
-            --cache-from=lsstsqre/gafaelfawr:${{steps.vars.outputs.tag}} \
-            --tag lsstsqre/gafaelfawr:${{steps.vars.outputs.tag}} .
-
-      - name: Push Docker images
-        run: |
-          docker push lsstsqre/gafaelfawr:deps-${{steps.vars.outputs.tag}}
-          docker push lsstsqre/gafaelfawr:${{steps.vars.outputs.tag}}
-
-  docs:
-    runs-on: ubuntu-latest
-
-    steps:
-      - uses: actions/checkout@v2.3.4
-
-      - name: Set up Python
-        uses: actions/setup-python@v2.1.4
+      - name: Cache Docker layers
+        uses: actions/cache@v2
         with:
-          python-version: 3.8
-
-      - name: Install tox and LTD Conveyor
-        run: pip install tox ltd-conveyor
+          path: /tmp/.buildx-cache
+          key: ${{ runner.os }}-buildx-${{ github.sha }}
+          restore-keys:
+            ${{ runner.os }}-buildx-
 
-      - name: Install graphviz and ImageMagick
-        run: sudo apt-get install graphviz imagemagick
-
-      - name: Run tox
-        run: tox -e docs
+      - name: Log in to Docker Hub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_TOKEN }}
 
-      - name: Upload to LSST the Docs
-        env:
-          LTD_USERNAME: ${{ secrets.LTD_USERNAME }}
-          LTD_PASSWORD: ${{ secrets.LTD_PASSWORD }}
-        run: ltd upload --product gafaelfawr --gh --dir docs/_build/html
+      - name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          push: true
+          tags: lsstsqre/gafaelfawr:${{ steps.vars.outputs.tag }}
+          cache-from: type=local,src=/tmp/.buildx-cache
+          cache-to: type=local,dest=/tmp/.buildx-cache-new
+
+      # Temp fix
+      # https://github.com/docker/build-push-action/issues/252
+      # https://github.com/moby/buildkit/issues/1896
+      - name: Move cache
+        run: |
+          rm -rf /tmp/.buildx-cache
+          mv /tmp/.buildx-cache-new /tmp/.buildx-cache