merge: #12424

12424: ci: always cache maven dependencies r=oleschoenburg a=oleschoenburg

This PR changes how we use and cache the local maven repository. This allows us to always build Zeebe in parallel and makes caching a more reliable.
To do this, we are using a new repository layout that supports concurrent builds and that splits release and snapshot artifacts. Splitting is very useful for us because we can now ensure that we only cache released artifacts and that locally built snapshot artifacts are not reused because that would be a correctness hazard.

Caching of maven dependencies is now always enabled but we split the caches between GH- and self-hosted (because they are not compatible with each other) and we use more cache modifiers to ensure that jobs that built only part of Zeebe don't create caches that are too small, i.e. contain only a few of Zeebe's dependencies.

Latest test run is here: https://github.com/camunda/zeebe/actions/runs/4916223809


Co-authored-by: Ole Schönburg <ole.schoenburg@gmail.com>

.github/actions/build-zeebe/action.yml

@@ -38,23 +38,8 @@ runs:
       name: Package Zeebe
       shell: bash
       id: build-java
-      # we do not build in parallel to avoid memory and cache corruption issues, notably observed
-      # on macOS and Windows
-      env:
-        # adds some additional Maven arguments; while the docs specify we should use MAVEN_ARGS with Maven 3.9, the Maven wrapper breaks this
-        # and instead uses its own MAVEN_CONFIG environment variable
-        #
-        # -e ensures errors will also spit out a stack trace, which is always useful, and has no impact on normal builds
-        # maven.wagon.* and maven.resolver.transport set the resolver's network transport to Wagon,
-        # the old provider pre 3.9. Until Maven 3.9.2, we have to do this if we want to retry on
-        # network issues, as otherwise any issue will fail the build.
-        MAVEN_CONFIG: >
-          -e
-          -D maven.wagon.httpconnectionManager.ttlSeconds=120 -D maven.wagon.http.pool=false -Dmaven.resolver.transport=wagon
-          -D maven.wagon.http.retryHandler.class=standard -D maven.wagon.http.retryHandler.requestSentEnabled=true
-          -D maven.wagon.http.retryHandler.count=5
       run: |
-        ./mvnw -B -DskipTests -DskipChecks install ${{ inputs.maven-extra-args }}
+        ./mvnw -B -T1C -DskipTests -DskipChecks install ${{ inputs.maven-extra-args }}
         export BUILD_DIR=$(./mvnw -pl dist/ help:evaluate -Dexpression=project.build.directory -q -DforceStdout)
         export ARTIFACT=$(./mvnw -pl dist/ help:evaluate -Dexpression=project.build.finalName -q -DforceStdout)
         echo "distball=${BUILD_DIR}/${ARTIFACT}.tar.gz" >> $GITHUB_OUTPUT

.github/actions/setup-zeebe/action.yml

@@ -18,13 +18,9 @@ inputs:
     description: The JDK version to setup
     default: "17"
     required: false
-  maven-cache:
-    description: A modifier key used to toggle the usage of a maven repo cache.
-    default: "false"
-    required: false
   maven-cache-key-modifier:
     description: A modifier key used for the maven cache, can be used to create isolated caches for certain jobs.
-    default: "default"
+    default: "shared"
     required: false
   secret_vault_address:
     description: 'secret vault url'
@@ -67,8 +63,8 @@ runs:
         secrets: |
           secret/data/products/zeebe/ci/zeebe ARTIFACTS_USR;
           secret/data/products/zeebe/ci/zeebe ARTIFACTS_PSW;
-    - if: ${{ inputs.java == 'true' }}
-      uses: actions/setup-java@v3
+    - uses: actions/setup-java@v3
+      if: inputs.java == 'true'
       with:
         distribution: 'temurin'
         java-version: ${{ inputs.java-version }}
@@ -79,7 +75,8 @@ runs:
     - name: 'Create settings.xml'
       uses: s4u/maven-settings-action@v2.8.0
       if: |
-        inputs.secret_vault_address != ''
+        inputs.java == 'true'
+        && inputs.secret_vault_address != ''
         && inputs.secret_vault_roleId != ''
         && inputs.secret_vault_secretId != ''
       with:
@@ -91,14 +88,61 @@ runs:
             "password": "${{ steps.secrets.outputs.ARTIFACTS_PSW }}"
           }]
         mirrors: '[{"url": "https://repository.nexus.camunda.cloud/content/groups/internal/", "id": "camunda-nexus", "mirrorOf": "zeebe,zeebe-snapshots", "name": "camunda Nexus"}]'
-    - if: ${{ inputs.maven-cache == 'true' }}
-      name: Cache local Maven repository
+    - name: Configure Maven
+      if: inputs.java == 'true'
+      shell: bash
+      # `--errors` ensures errors will also spit out a stack trace, which is always useful, and has no impact on normal builds
+      #
+      # `--update-snapshots` to force Maven into updating snapshots, but also to retry looking for
+      #    release artifacts when an earlier lookup failure made it into the cache.
+      #
+      # `maven.wagon.*` and `maven.resolver.transport` set the resolver's network transport to Wagon,
+      #    the old provider pre 3.9. Until Maven 3.9.2, we have to do this if we want to retry on
+      #    network issues, as otherwise any issue will fail the build.
+      #
+      # `aether.enhancedLocalRepository.split` splits between local and remote artifacts.
+      # `aether.enhancedLocalRepository.splitRemote` splits remote artifacts into released and snapshot
+      # `aether.syncContext.*` config ensures that maven uses file locks to prevent corruption
+      #      from downloading multiple artifacts at the same time.
+      run: |
+        tee .mvn/maven.config <<EOF
+        --errors
+        --batch-mode
+        --update-snapshots
+        -D maven.wagon.httpconnectionManager.ttlSeconds=120
+        -D maven.wagon.http.pool=false
+        -D maven.resolver.transport=wagon
+        -D maven.wagon.http.retryHandler.class=standard
+        -D maven.wagon.http.retryHandler.requestSentEnabled=true
+        -D maven.wagon.http.retryHandler.count=5
+        -D aether.enhancedLocalRepository.split=true
+        -D aether.enhancedLocalRepository.splitRemote=true
+        -D aether.syncContext.named.nameMapper=file-gav
+        -D aether.syncContext.named.factory=file-lock
+        -D maven.artifact.threads=32
+        EOF
+    - name: Cache local Maven repository
+      if: inputs.java == 'true' && startsWith(runner.name, 'actions-runner-')
+      uses: actions/cache@v3
+      with:
+        # This is the path used by the `enhancedLocalRepository` set up in the 'Configure Maven' step.
+        # `aether.enhancedLocalRepository.remotePrefix` defaults to 'cached'
+        # `aether.enhancedLocalRepository.releasesPrefix` defaults to 'releases'
+        path: ~/.m2/repository/cached/releases/
+        key: self-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          self-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}
+    - name: Cache local Maven repository
+      if: inputs.java == 'true' && !startsWith(runner.name, 'actions-runner-')
       uses: actions/cache@v3
       with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-maven-${{ inputs.maven-cache-key-modifier }}-${{ hashFiles('**/pom.xml') }}
+        # This is the path used by the `enhancedLocalRepository` set up in the 'Configure Maven' step.
+        # `aether.enhancedLocalRepository.remotePrefix` defaults to 'cached'
+        # `aether.enhancedLocalRepository.releasesPrefix` defaults to 'releases'
+        path: ~/.m2/repository/cached/releases/
+        key: gh-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
-          ${{ runner.os }}-maven-${{ inputs.maven-cache-key-modifier }}-
+          gh-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}
     - if: ${{ inputs.go == 'true' }}
       uses: actions/setup-go@v3
       with:

.github/workflows/ci.yml

@@ -66,7 +66,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-zeebe
         with:
-          maven-cache: 'true'
+          maven-cache-key-modifier: it-${{ matrix.group }}
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -118,7 +118,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -216,7 +215,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -283,7 +281,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           maven-cache-key-modifier: java-client
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
@@ -330,7 +327,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -385,7 +381,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           maven-cache-key-modifier: java-checks
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}