ci: Fix test coverage in CI.

.github/workflows/ci.yml

@@ -49,13 +49,23 @@ jobs:
         run: poetry export --with=dev --output docker/requirements.txt && git diff --exit-code
       - name: Build python packages
         run: poetry build
+      - name: SonarCloud Scan
+        uses: SonarSource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
   tests:
     name: Run tests in Glue container
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.1.5
       - name: Build container and run tests
-        run: docker compose --file docker/docker-compose.yml run --rm --remove-orphans --build glue-utils -c 'pytest -p no:cacheprovider'
+        run: USER_ID=$(id -u) docker compose --file docker/docker-compose.yml run --rm --remove-orphans --build glue-utils -c pytest
+      - name: SonarCloud Scan
+        uses: SonarSource/sonarcloud-github-action@master
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
+          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}
   pip-audit:
     name: Audit requirements
     runs-on: ubuntu-latest

Makefile

@@ -1,3 +1,5 @@
+DOCKER_COMPOSE_RUN = USER_ID=$$(id -u) docker compose --file docker/docker-compose.yml run --rm --build glue-utils
+
 .PHONY: all
 all: ## Show help (default)
 	@echo "=== Glue Utils ==="
@@ -45,17 +47,12 @@ importcheck: ## Check import rules
 
 .PHONY: test
 test: docker/requirements.txt ## Run automated tests
-	@docker compose --file docker/docker-compose.yml run --rm --build glue-utils -c pytest
-
-
-.PHONY: coverage
-coverage: docker/requirements.txt ## Generate test coverage HTML report
-	@docker compose --file docker/docker-compose.yml run --rm --build glue-utils -c "pytest --cov=glue_utils --cov-branch --cov-report=term --cov-report=html"
+	@$(DOCKER_COMPOSE_RUN) -c pytest
 
 
 .PHONY: shell
 shell: docker/requirements.txt ## Enter a shell in the container
-	@docker compose --file docker/docker-compose.yml run --rm --build glue-utils -c bash
+	@$(DOCKER_COMPOSE_RUN) -c bash
 
 
 .PHONY: checks

README.md

@@ -2,6 +2,7 @@
 
 [![License](https://img.shields.io/badge/license-MIT-blue.svg)](LICENSE)
 [![Quality Gate Status](https://sonarcloud.io/api/project_badges/measure?project=dashmug_glue-utils&metric=alert_status)](https://sonarcloud.io/summary/new_code?id=dashmug_glue-utils)
+[![Coverage](https://sonarcloud.io/api/project_badges/measure?project=dashmug_glue-utils&metric=coverage)](https://sonarcloud.io/summary/new_code?id=dashmug_glue-utils)
 [![Reliability Rating](https://sonarcloud.io/api/project_badges/measure?project=dashmug_glue-utils&metric=reliability_rating)](https://sonarcloud.io/summary/new_code?id=dashmug_glue-utils)
 [![Security Rating](https://sonarcloud.io/api/project_badges/measure?project=dashmug_glue-utils&metric=security_rating)](https://sonarcloud.io/summary/new_code?id=dashmug_glue-utils)
 [![Maintainability Rating](https://sonarcloud.io/api/project_badges/measure?project=dashmug_glue-utils&metric=sqale_rating)](https://sonarcloud.io/summary/new_code?id=dashmug_glue-utils)

docker/Dockerfile

@@ -1,12 +1,30 @@
 FROM amazon/aws-glue-libs:glue_libs_4.0.0_image_01
 
+# Pass the host user to the container.
+ARG USER_ID
+
+# Switch to root to be able to make changes in the container filesystem.
+USER root
+
+# Change UID of glue_user to be the same as host user. This allows
+# JupyterLab to write to the host system as glue_user.
+RUN usermod -u "$USER_ID" glue_user \
+  # Clean up /tmp which may already have glue_user-owned files with the
+  # old UID.
+  && rm -rf /tmp/*
+
+# Switch to glue_user to be able to make changes for the user itself.
+USER glue_user
+
 # Copy requirements file that contains tooling.
 WORKDIR /home/glue_user/workspace
 COPY docker/requirements.txt ./
 
 RUN pip3 install --no-cache-dir --no-warn-script-location --user --upgrade pip==24.0 \
   # Install dev requirements.
-  && pip3 install --no-cache-dir --no-warn-script-location --user -r requirements.txt
+  && pip3 install --no-cache-dir --no-warn-script-location --user -r requirements.txt \
+  # Prepare a /tmp directory needed by Spark to start.
+  && mkdir -p /tmp/spark-events
 
 COPY src pyproject.toml README.md ./
 

docker/docker-compose.yml

@@ -6,6 +6,8 @@ services:
     build:
       context: ../
       dockerfile: docker/Dockerfile
+      args:
+        - USER_ID=${USER_ID:-}
     volumes:
       - ..:/home/glue_user/workspace
     tty: true

docker/requirements.txt

@@ -225,6 +225,6 @@ tomli==2.0.1 ; python_version >= "3.9" and python_full_version <= "3.11.0a6" \
 typing-extensions==4.11.0 ; python_version >= "3.9" and python_version < "4.0" \
     --hash=sha256:83f085bd5ca59c80295fc2a82ab5dac679cbe02b9f33f7d83af68e241bea51b0 \
     --hash=sha256:c1f94d72897edaf4ce775bb7558d5b79d8126906a14ea5ed1635921406c0387a
-zipp==3.18.1 ; python_version >= "3.9" and python_version < "3.10" \
-    --hash=sha256:206f5a15f2af3dbaee80769fb7dc6f249695e940acca08dfb2a4769fe61e538b \
-    --hash=sha256:2884ed22e7d8961de1c9a05142eb69a247f120291bc0206a00a7642f09b5b715
+zipp==3.18.2 ; python_version >= "3.9" and python_version < "3.10" \
+    --hash=sha256:6278d9ddbcfb1f1089a88fde84481528b07b0e10474e09dcfe53dad4069fa059 \
+    --hash=sha256:dce197b859eb796242b0622af1b8beb0a722d52aa2f57133ead08edd5bf5374e

pyproject.toml

@@ -103,6 +103,7 @@ disallow_incomplete_defs = false
 disallow_untyped_defs = false
 
 [tool.pytest.ini_options]
+addopts = "--cov=glue_utils --cov-branch --cov-report=term --cov-report=html --cov-report=xml"
 filterwarnings = [
     "ignore::FutureWarning:pyspark.sql.context",
 ]
@@ -129,6 +130,9 @@ exclude_also = [
     "@overload",
 ]
 
+[tool.coverage.xml]
+output = "coverage/results.xml"
+
 [tool.bumpver]
 current_version = "0.4.0"
 version_pattern = "MAJOR.MINOR.PATCH[PYTAGNUM]"

sonar-project.properties

@@ -4,4 +4,5 @@ sonar.organization=dashmug
 sonar.sources=src
 sonar.tests=test
 sonar.pullrequest.github.summary_comment=true
-sonar.python.version=3.9,3.10
+sonar.python.version=3.9,3.10
+sonar.python.coverage.reportPaths=coverage/results.xml