Allow a broader range of distro pytest versions

.github/workflows/ci.yml

@@ -1,11 +1,15 @@
 name: CI
 on: [push]
+
+env:
+  CACHE_BUST_ITER: 0
+
 jobs:
   tests:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        python-version: [3.7, 3.8, 3.9]
+        python-version: ["3.8", "3.9", "3.10", "3.11"]
 
     # Service containers to run
     services:
@@ -39,17 +43,27 @@ jobs:
       - name: Apt install
         run: sudo apt-get install -y libxmlsec1-dev
 
+      - name: Initialize Pants
+        uses: pantsbuild/actions/init-pants@7151afe1e66338457f7f65a894bc12a2d8f9cab7
+        with:
+          # cache0 makes it easy to bust the cache if needed
+          gha-cache-key: cache-${{ env.CACHE_BUST_ITER }}-py${{ matrix.python_version }}
+          named-caches-hash: ${{ hashFiles('build-support/lockfiles/*.lockfile') }}
+
+      - name: Prune pants cache
+        run: bash ./.github/workflows/scripts/prune_pants_cache.sh
+
       - name: Lints
-        run: './pants lint ::'
+        run: 'pants lint ::'
 
       # - name: Typecheck
-      #   run: './pants typecheck ::'
+      #   run: 'pants typecheck ::'
 
       - name: Tests
-        run: './pants test ::'
+        run: 'pants test ::'
 
       - name: Changelog
-        run: './pants ol-project changelog check'
+        run: 'pants ol-project changelog check'
 
       # - name: Upload coverage to CodeCov
       #   uses: codecov/codecov-action@v1
@@ -78,7 +92,7 @@ jobs:
           core.setOutput('app-package', appName.replace("mitol-django-", "").replace(/-/g, "_"))
 
     - name: Build
-      run: ./pants package "src/mitol/${{steps.get-app-name.outputs.app-package}}:"
+      run: pants package "src/mitol/${{steps.get-app-name.outputs.app-package}}:"
 
     - uses: pypa/gh-action-pypi-publish@release/v1
       with:

.github/workflows/scripts/prune_pants_cache.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+function prune_if_too_big() {
+   path=$1
+   limit_mb=$2
+   size_mb=$(du -m -d0 "${path}" | cut -f 1)
+   if (( size_mb > limit_mb )); then
+     echo "${path} is too large (${size_mb}mb), pruning it."
+     rm -rf "${path}"
+   fi
+ }
+
+prune_if_too_big ~/.cache/pants/setup 512
+prune_if_too_big ~/.cache/pants/named_caches 1024

Dockerfile

@@ -0,0 +1,75 @@
+# syntax=docker/dockerfile:1
+FROM ubuntu:latest as base
+
+SHELL ["bash", "-l", "-c"]
+ENV SHELL=/bin/bash
+
+RUN ln -fs /usr/share/zoneinfo/UTC /etc/localtime
+RUN <<EOT bash
+    export DEBIAN_FRONTEND=noninteractive
+    apt-get update
+    apt-get install -y curl git build-essential pkg-config gdb lcov pkg-config \
+      libbz2-dev libffi-dev libgdbm-dev libgdbm-compat-dev liblzma-dev \
+      libncurses5-dev libreadline6-dev libsqlite3-dev libssl-dev \
+      lzma lzma-dev tk-dev uuid-dev zlib1g-dev
+EOT
+
+WORKDIR /tmp
+COPY apt.txt /tmp/apt.txt
+RUN xargs apt-get install -y <apt.txt
+
+RUN useradd dev
+USER dev
+WORKDIR /home/dev
+
+# ===================================================================
+FROM base as pyenv
+
+RUN curl https://pyenv.run | bash
+RUN <<EOT bash
+    echo 'export PYENV_ROOT="\$HOME/.pyenv"' >> ~/.bashrc
+    echo 'command -v pyenv >/dev/null || export PATH="\$PYENV_ROOT/bin:\$PATH"' >> ~/.bashrc
+    echo 'eval "\$(pyenv init -)"' >> ~/.bashrc
+    echo 'export PYENV_ROOT="\$HOME/.pyenv"' >> ~/.profile
+    echo 'command -v pyenv >/dev/null || export PATH="\$PYENV_ROOT/bin:\$PATH"' >> ~/.profile
+    echo 'eval "\$(pyenv init -)"' >> ~/.profile
+EOT
+
+# ===================================================================
+FROM pyenv as py38
+RUN pyenv install 3.8
+
+# ===================================================================
+FROM pyenv as py39
+RUN pyenv install 3.9
+
+# ===================================================================
+FROM pyenv as py310
+RUN pyenv install 3.10
+
+# ===================================================================
+FROM pyenv as py311
+RUN pyenv install 3.11
+
+# ===================================================================
+FROM base as pants
+RUN curl --proto '=https' --tlsv1.2 -fsSL https://static.pantsbuild.org/setup/get-pants.sh | bash
+
+# ===================================================================
+FROM pyenv as shell
+
+ENV PATH="${PATH}:/home/dev/bin"
+
+# the pants installer puts things in ~/cache/nce and it needs to be persistent
+RUN mkdir -p .cache && chown dev:dev .cache
+
+COPY --from=pants /home/dev/bin ./bin
+
+# these are all separate stages to make them build in parallel
+COPY --from=py38 /home/dev/.pyenv/versions ./.pyenv/versions/
+COPY --from=py39 /home/dev/.pyenv/versions ./.pyenv/versions/
+COPY --from=py310 /home/dev/.pyenv/versions ./.pyenv/versions/
+COPY --from=py311 /home/dev/.pyenv/versions ./.pyenv/versions/
+
+VOLUME /home/dev/.cache
+WORKDIR /home/dev/src

README.md

@@ -6,7 +6,7 @@ This repository is the home of MIT Open Learning's reusable django apps
 
 We maintain changelogs in `changelog.d/` directories with each app. To create a new changelog for your changes, run:
 
-- `/pants ol-project changelog create --app APPNAME`
+- `pants ol-project changelog create --app APPNAME`
   - `APPNAME`: the name of an application directory
 
 Then fill out the new file that was generated with information about your changes. These changes will all be merged down into `CHANGELOG.md` when a release is generated.
@@ -17,7 +17,7 @@ Changelogs are maintained according to [Keep a Changelog](https://keepachangelog
 Versioning uses a date-based versioning scheme with incremental builds on the same day.
 Version tags follow `{package-name}/v{version`
 To perform a release, run:
-- `./pants ol-project release create --app APPNAME --push`:
+- `pants ol-project release create --app APPNAME --push`:
   - `APPNAME`: the name of an application directory
 
 ### Navigating this repository
@@ -29,7 +29,17 @@ To perform a release, run:
 
 ### Prerequisites
 
+#### Use the docker-compose container (recommended)
+
+- Run `docker compose run --rm shell bash` to get a clean sandbox environment
+
+#### Use on your host system
+
 - Install `xmlsec` native libraries for your OS: https://xmlsec.readthedocs.io/en/stable/install.html
+- Install `pants` (this is actually `scie-pants`, a context-aware wrapper script that bootstraps the correct `pants` version and its depenencies):
+  - Use their installer script: https://www.pantsbuild.org/docs/installation
+  - Alternatively, if you don't want to pipe a script from the internet directly into `bash`, you can download the latest release of `scie-pants` for your os/arch and put it somewhere on your `PATH` (the installer script puts it in `~/bin`): https://github.com/pantsbuild/scie-pants/releases
+
 
 ### Usage
 
@@ -40,32 +50,32 @@ We use [`pants`](https://www.pantsbuild.org/) to manage apps and releases.
 Useful commands:
 ```shell
 # run all tests
-./pants test ::
+pants test ::
 # run only common app tests
-./pants test tests/mitol/common:
+pants test tests/mitol/common:
 
 # format code (isort + black)
-./pants fmt ::
+pants fmt ::
 
 # run lints
-./pants lint ::
+pants lint ::
 
 # run django management scripts
-./pants run tests/manage.py -- ARGS
+pants run tests/manage.py -- ARGS
 # run a django shell
-./pants run tests/manage.py -- shell  
+pants run tests/manage.py -- shell  
 # create migrations
-./pants run tests/manage.py -- makemigrations  
+pants run tests/manage.py -- makemigrations  
 # run a django migrate
-./pants run tests/manage.py -- migrate  
+pants run tests/manage.py -- migrate  
 ```
 
 ### Migrations
 
 To generate migrations for a reusable app, run the standard:
 
 ```
-./pants run tests/manage.py -- makemigrations APP_NAME --name MIGRATION_NAME
+pants run tests/manage.py -- makemigrations APP_NAME --name MIGRATION_NAME
 ```
 
 where `APP_NAME` matches the `name` attribute from your `apps.py` app.
@@ -76,7 +86,7 @@ where `APP_NAME` matches the `name` attribute from your `apps.py` app.
 ```shell
 NAME="[APP-NAME]"
 mkdir src/mitol/$NAME
-./pants run :django-admin -- startapp $NAME src/mitol/$NAME
+pants run :django-admin -- startapp $NAME src/mitol/$NAME
 ```
 - Add a `BUILD` file, following using the same files from other projects as a guideline
 - Add the project as a dependency in `src/BUILD`

apt.txt

@@ -0,0 +1,3 @@
+libxml2-dev
+libxmlsec1-dev
+libxmlsec1-openssl

docker-compose.yml

@@ -9,3 +9,17 @@ services:
       POSTGRES_PASSWORD: postgres
     ports:
       - "5432:5432"
+
+  shell:
+    build:
+      target: shell
+    volumes:
+      - .:/home/dev/src
+      - cache:/home/dev/.cache/
+    environment:
+      DATABASE_URL: postgres://postgres:postgres@db:5432/postgres
+    depends_on:
+      - db
+
+volumes:
+  cache: