merge: #12726

12726: [Backport stable/8.2] ci: always cache maven dependencies r=oleschoenburg a=backport-action # Description Backport of #12424 to `stable/8.2`. relates to Co-authored-by: Ole Schönburg <ole.schoenburg@gmail.com>
camunda · May 9, 2023 · 7531996 · 7531996
2 parents 8bffe0c + ac0fcfc
commit 7531996
Show file tree

Hide file tree

Showing 3 changed files with 59 additions and 35 deletions.
diff --git a/.github/actions/build-zeebe/action.yml b/.github/actions/build-zeebe/action.yml
@@ -38,23 +38,8 @@ runs:
       name: Package Zeebe
       shell: bash
       id: build-java
-      # we do not build in parallel to avoid memory and cache corruption issues, notably observed
-      # on macOS and Windows
-      env:
-        # adds some additional Maven arguments; while the docs specify we should use MAVEN_ARGS with Maven 3.9, the Maven wrapper breaks this
-        # and instead uses its own MAVEN_CONFIG environment variable
-        #
-        # -e ensures errors will also spit out a stack trace, which is always useful, and has no impact on normal builds
-        # maven.wagon.* and maven.resolver.transport set the resolver's network transport to Wagon,
-        # the old provider pre 3.9. Until Maven 3.9.2, we have to do this if we want to retry on
-        # network issues, as otherwise any issue will fail the build.
-        MAVEN_CONFIG: >
-          -e
-          -D maven.wagon.httpconnectionManager.ttlSeconds=120 -D maven.wagon.http.pool=false -Dmaven.resolver.transport=wagon
-          -D maven.wagon.http.retryHandler.class=standard -D maven.wagon.http.retryHandler.requestSentEnabled=true
-          -D maven.wagon.http.retryHandler.count=5
       run: |
-        ./mvnw -B -DskipTests -DskipChecks install ${{ inputs.maven-extra-args }}
+        ./mvnw -B -T1C -DskipTests -DskipChecks install ${{ inputs.maven-extra-args }}
         export BUILD_DIR=$(./mvnw -pl dist/ help:evaluate -Dexpression=project.build.directory -q -DforceStdout)
         export ARTIFACT=$(./mvnw -pl dist/ help:evaluate -Dexpression=project.build.finalName -q -DforceStdout)
         echo "distball=${BUILD_DIR}/${ARTIFACT}.tar.gz" >> $GITHUB_OUTPUT
diff --git a/.github/actions/setup-zeebe/action.yml b/.github/actions/setup-zeebe/action.yml
@@ -18,13 +18,9 @@ inputs:
     description: The JDK version to setup
     default: "17"
     required: false
-  maven-cache:
-    description: A modifier key used to toggle the usage of a maven repo cache.
-    default: "false"
-    required: false
   maven-cache-key-modifier:
     description: A modifier key used for the maven cache, can be used to create isolated caches for certain jobs.
-    default: "default"
+    default: "shared"
     required: false
   secret_vault_address:
     description: 'secret vault url'
@@ -67,8 +63,8 @@ runs:
         secrets: |
           secret/data/products/zeebe/ci/zeebe ARTIFACTS_USR;
           secret/data/products/zeebe/ci/zeebe ARTIFACTS_PSW;
-    - if: ${{ inputs.java == 'true' }}
-      uses: actions/setup-java@v3
+    - uses: actions/setup-java@v3
+      if: inputs.java == 'true'
       with:
         distribution: 'temurin'
         java-version: ${{ inputs.java-version }}
@@ -79,7 +75,8 @@ runs:
     - name: 'Create settings.xml'
       uses: s4u/maven-settings-action@v2.8.0
       if: |
-        inputs.secret_vault_address != ''
+        inputs.java == 'true'
+        && inputs.secret_vault_address != ''
         && inputs.secret_vault_roleId != ''
         && inputs.secret_vault_secretId != ''
       with:
@@ -91,14 +88,61 @@ runs:
             "password": "${{ steps.secrets.outputs.ARTIFACTS_PSW }}"
           }]
         mirrors: '[{"url": "https://repository.nexus.camunda.cloud/content/groups/internal/", "id": "camunda-nexus", "mirrorOf": "zeebe,zeebe-snapshots", "name": "camunda Nexus"}]'
-    - if: ${{ inputs.maven-cache == 'true' }}
-      name: Cache local Maven repository
+    - name: Configure Maven
+      if: inputs.java == 'true'
+      shell: bash
+      # `--errors` ensures errors will also spit out a stack trace, which is always useful, and has no impact on normal builds
+      #
+      # `--update-snapshots` to force Maven into updating snapshots, but also to retry looking for
+      #    release artifacts when an earlier lookup failure made it into the cache.
+      #
+      # `maven.wagon.*` and `maven.resolver.transport` set the resolver's network transport to Wagon,
+      #    the old provider pre 3.9. Until Maven 3.9.2, we have to do this if we want to retry on
+      #    network issues, as otherwise any issue will fail the build.
+      #
+      # `aether.enhancedLocalRepository.split` splits between local and remote artifacts.
+      # `aether.enhancedLocalRepository.splitRemote` splits remote artifacts into released and snapshot
+      # `aether.syncContext.*` config ensures that maven uses file locks to prevent corruption
+      #      from downloading multiple artifacts at the same time.
+      run: |
+        tee .mvn/maven.config <<EOF
+        --errors
+        --batch-mode
+        --update-snapshots
+        -D maven.wagon.httpconnectionManager.ttlSeconds=120
+        -D maven.wagon.http.pool=false
+        -D maven.resolver.transport=wagon
+        -D maven.wagon.http.retryHandler.class=standard
+        -D maven.wagon.http.retryHandler.requestSentEnabled=true
+        -D maven.wagon.http.retryHandler.count=5
+        -D aether.enhancedLocalRepository.split=true
+        -D aether.enhancedLocalRepository.splitRemote=true
+        -D aether.syncContext.named.nameMapper=file-gav
+        -D aether.syncContext.named.factory=file-lock
+        -D maven.artifact.threads=32
+        EOF
+    - name: Cache local Maven repository
+      if: inputs.java == 'true' && startsWith(runner.name, 'actions-runner-')
+      uses: actions/cache@v3
+      with:
+        # This is the path used by the `enhancedLocalRepository` set up in the 'Configure Maven' step.
+        # `aether.enhancedLocalRepository.remotePrefix` defaults to 'cached'
+        # `aether.enhancedLocalRepository.releasesPrefix` defaults to 'releases'
+        path: ~/.m2/repository/cached/releases/
+        key: self-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}-${{ hashFiles('**/pom.xml') }}
+        restore-keys: |
+          self-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}
+    - name: Cache local Maven repository
+      if: inputs.java == 'true' && !startsWith(runner.name, 'actions-runner-')
       uses: actions/cache@v3
       with:
-        path: ~/.m2/repository
-        key: ${{ runner.os }}-maven-${{ inputs.maven-cache-key-modifier }}-${{ hashFiles('**/pom.xml') }}
+        # This is the path used by the `enhancedLocalRepository` set up in the 'Configure Maven' step.
+        # `aether.enhancedLocalRepository.remotePrefix` defaults to 'cached'
+        # `aether.enhancedLocalRepository.releasesPrefix` defaults to 'releases'
+        path: ~/.m2/repository/cached/releases/
+        key: gh-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}-${{ hashFiles('**/pom.xml') }}
         restore-keys: |
-          ${{ runner.os }}-maven-${{ inputs.maven-cache-key-modifier }}-
+          gh-hosted-${{ runner.os }}-mvn-${{ inputs.maven-cache-key-modifier }}
     - if: ${{ inputs.go == 'true' }}
       uses: actions/setup-go@v3
       with:

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -66,7 +66,7 @@ jobs:
       - uses: actions/checkout@v3
       - uses: ./.github/actions/setup-zeebe
         with:
-          maven-cache: 'true'
+          maven-cache-key-modifier: it-${{ matrix.group }}
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -118,7 +118,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -216,7 +215,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -286,7 +284,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           maven-cache-key-modifier: java-client
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
@@ -333,7 +330,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}
           secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
@@ -388,7 +384,6 @@ jobs:
       - uses: ./.github/actions/setup-zeebe
         with:
           go: false
-          maven-cache: 'true'
           maven-cache-key-modifier: java-checks
           secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
           secret_vault_address: ${{ secrets.VAULT_ADDR }}