build_release_candidate #81
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: build_release_candidate | |
| # Workflow added after https://github.com/apache/beam/commit/4183e747becebd18becee5fff547af365910fc9c | |
| # If help is needed debugging issues, you can view the release guide at that commit for guidance on how to do this manually. | |
| # (https://github.com/apache/beam/blob/4183e747becebd18becee5fff547af365910fc9c/website/www/site/content/en/contribute/release-guide.md) | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| RELEASE: | |
| description: Beam version of current release (e.g. 2.XX.0) | |
| required: true | |
| default: '2.XX.0' | |
| RC: | |
| description: Integer RC version for the release (e.g. 3 for RC3) | |
| required: true | |
| APACHE_ID: | |
| description: Your Apache id. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/ | |
| required: false | |
| APACHE_PASSWORD: | |
| description: Your Apache password. Required if you want to stage artifacts into https://dist.apache.org/repos/dist/dev/beam/ | |
| required: false | |
| BEAM_SITE_TOKEN: | |
| description: Github Personal Access Token with apache/beam-site repo permission if you want to create the beam-site docs PR. See https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens. | |
| default: '' | |
| PUBLISH_JAVA_ARTIFACTS: | |
| description: Whether to publish java artifacts to https://repository.apache.org/#stagingRepositories (yes/no) | |
| required: true | |
| default: 'no' | |
| STAGE_JAVA_SOURCE: | |
| description: Whether to stage the java source into https://dist.apache.org/repos/dist/dev/beam/ | |
| required: true | |
| default: 'no' | |
| STAGE_DOCKER_ARTIFACTS: | |
| description: Whether to stage SDK docker images to docker hub Apache organization | |
| required: true | |
| default: 'no' | |
| STAGE_PYTHON_ARTIFACTS: | |
| description: Whether to stage the python artifacts into https://dist.apache.org/repos/dist/dev/beam/ | |
| required: true | |
| default: 'no' | |
| CREATE_BEAM_SITE_PR: | |
| description: Whether to create the documentation update PR against apache/beam-site. | |
| required: true | |
| default: 'no' | |
| env: | |
| GRADLE_ENTERPRISE_ACCESS_KEY: ${{ secrets.GE_ACCESS_TOKEN }} | |
| GRADLE_ENTERPRISE_CACHE_USERNAME: ${{ secrets.GE_CACHE_USERNAME }} | |
| GRADLE_ENTERPRISE_CACHE_PASSWORD: ${{ secrets.GE_CACHE_PASSWORD }} | |
| jobs: | |
| publish_java_artifacts: | |
| if: ${{github.event.inputs.PUBLISH_JAVA_ARTIFACTS == 'yes'}} | |
| runs-on: [self-hosted, ubuntu-20.04, main] | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" | |
| repository: apache/beam | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '8' | |
| - name: Import GPG key | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| - name: Auth for nexus | |
| run: | | |
| mkdir -p ${HOME}/.m2 | |
| echo "<settings> | |
| <servers> | |
| <server> | |
| <id>apache.releases.https</id> | |
| <username>${{ secrets.NEXUS_STAGE_DEPLOYER_USER }}</username> | |
| <password>${{ secrets.NEXUS_STAGE_DEPLOYER_PW }}</password> | |
| </server> | |
| <server> | |
| <id>apache.snapshots.https</id> | |
| <username>${{ secrets.NEXUS_USER }}</username> | |
| <password>${{ secrets.NEXUS_PW }}</password> | |
| </server> | |
| </servers> | |
| </settings>" > ${HOME}/.m2/settings.xml | |
| - name: Configure git | |
| run: git config credential.helper store | |
| - name: Stage Java Artifacts into Maven | |
| run: ./gradlew publish -Psigning.gnupg.keyName=${{steps.import_gpg.outputs.fingerprint}} -PisRelease --no-daemon --no-parallel | |
| stage_java_source: | |
| if: ${{github.event.inputs.STAGE_JAVA_SOURCE == 'yes'}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Validate and mask apache id/password | |
| run: | | |
| echo "::add-mask::${{ github.event.inputs.APACHE_PASSWORD }}" | |
| if [ "${{ github.event.inputs.APACHE_ID }}" == "" ] | |
| then | |
| echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" | |
| fi | |
| if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ] | |
| then | |
| echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" | |
| fi | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '8' | |
| - name: Import GPG key | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| - name: stage source | |
| run: | | |
| svn co https://dist.apache.org/repos/dist/dev/beam | |
| mkdir -p beam/${{ github.event.inputs.RELEASE }} | |
| cd beam/${{ github.event.inputs.RELEASE }} | |
| RC_DIR="beam-${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" | |
| RC_ZIP="${RC_DIR}.zip" | |
| RELEASE_DIR="beam-${{ github.event.inputs.RELEASE }}" | |
| RC_TAG="v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" | |
| SOURCE_RELEASE_ZIP="apache-beam-${{ github.event.inputs.RELEASE }}-source-release.zip" | |
| # Check whether there is an existing dist dir | |
| if (svn ls "${SOURCE_RELEASE_ZIP}"); then | |
| echo "Removing existing ${SOURCE_RELEASE_ZIP}." | |
| svn delete "${SOURCE_RELEASE_ZIP}" | |
| fi | |
| echo "Downloading: https://github.com/apache/beam/archive/${RC_TAG}.zip" | |
| wget https://github.com/apache/beam/archive/${RC_TAG}.zip -O "${RC_ZIP}" | |
| unzip "$RC_ZIP" | |
| rm "$RC_ZIP" | |
| mv "$RC_DIR" "$RELEASE_DIR" | |
| zip -r "${SOURCE_RELEASE_ZIP}" "$RELEASE_DIR" | |
| rm -r "$RELEASE_DIR" | |
| echo "----Signing Source Release ${SOURCE_RELEASE_ZIP}-----" | |
| gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${SOURCE_RELEASE_ZIP}" | |
| echo "----Creating Hash Value for ${SOURCE_RELEASE_ZIP}----" | |
| sha512sum ${SOURCE_RELEASE_ZIP} > ${SOURCE_RELEASE_ZIP}.sha512 | |
| svn add --force . | |
| svn status | |
| svn commit -m "Staging Java artifacts for Apache Beam ${{ github.event.inputs.RELEASE }} RC${{ github.event.inputs.RC }}" --non-interactive --username ${{ github.event.inputs.APACHE_ID }} --password ${{ github.event.inputs.APACHE_PASSWORD }} | |
| stage_python_artifacts: | |
| if: ${{github.event.inputs.STAGE_PYTHON_ARTIFACTS == 'yes'}} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| - name: Validate and mask apache id/password | |
| run: | | |
| echo "::add-mask::${{ github.event.inputs.APACHE_PASSWORD }}" | |
| if [ "${{ github.event.inputs.APACHE_ID }}" == "" ] | |
| then | |
| echo "Must provide an apache id to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" | |
| fi | |
| if [ "${{ github.event.inputs.APACHE_PASSWORD }}" == "" ] | |
| then | |
| echo "Must provide an apache password to stage artifacts to https://dist.apache.org/repos/dist/dev/beam/" | |
| fi | |
| - name: Setup environment | |
| uses: ./.github/actions/setup-environment-action | |
| with: | |
| python-version: 3.8 | |
| - name: Import GPG key | |
| id: import_gpg | |
| uses: crazy-max/ghaction-import-gpg@111c56156bcc6918c056dbef52164cfa583dc549 | |
| with: | |
| gpg_private_key: ${{ secrets.GPG_PRIVATE_KEY }} | |
| - name: Install dependencies | |
| run: | | |
| pip install python-dateutil | |
| pip install requests | |
| - name: stage python artifacts | |
| env: | |
| RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" | |
| GIT_REPO_BASE_URL: https://github.com/apache/beam | |
| RELEASE_DIR: "beam/${{ github.event.inputs.RELEASE }}" | |
| RELEASE: "${{ github.event.inputs.RELEASE }}" | |
| SCRIPT_DIR: release/src/main/scripts | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| SVN_ARTIFACTS_DIR: "beam/${{ github.event.inputs.RELEASE }}/python" | |
| run: | | |
| svn co https://dist.apache.org/repos/dist/dev/beam | |
| mkdir -p "${SVN_ARTIFACTS_DIR}" | |
| git fetch --all --tags --prune | |
| RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") | |
| python "${SCRIPT_DIR}/download_github_actions_artifacts.py" \ | |
| --github-token-var GITHUB_TOKEN \ | |
| --repo-url "apache/beam" \ | |
| --rc-tag "${RC_TAG}" \ | |
| --release-commit "${RELEASE_COMMIT}" \ | |
| --artifacts_dir "${RELEASE_DIR}/python" \ | |
| --yes True | |
| cd "${RELEASE_DIR}"/python | |
| ls | |
| echo "------Checking Hash Value for apache-beam-${RELEASE}.tar.gz-----" | |
| sha512sum -c "apache-beam-${RELEASE}.tar.gz.sha512" | |
| echo "------Signing Source Release apache-beam-${RELEASE}.tar.gz------" | |
| gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --detach-sig "apache-beam-${RELEASE}.tar.gz" | |
| for artifact in *.whl; do | |
| echo "----------Checking Hash Value for ${artifact} wheel-----------" | |
| sha512sum -c "${artifact}.sha512" | |
| done | |
| for artifact in *.whl; do | |
| echo "------------------Signing ${artifact} wheel-------------------" | |
| gpg --local-user "${{steps.import_gpg.outputs.name}}" --armor --batch --yes --detach-sig "${artifact}" | |
| done | |
| cd .. | |
| svn add --force python | |
| svn status | |
| svn commit -m "Staging Python artifacts for Apache Beam ${RELEASE} RC${RC_NUM}" --non-interactive --username ${{ github.event.inputs.APACHE_ID }} --password ${{ github.event.inputs.APACHE_PASSWORD }} | |
| stage_docker: | |
| if: ${{github.event.inputs.STAGE_DOCKER_ARTIFACTS == 'yes'}} | |
| # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" | |
| repository: apache/beam | |
| - name: Free Disk Space (Ubuntu) | |
| uses: jlumbroso/free-disk-space@v1.3.0 | |
| - name: Install Java 11 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '11' | |
| - name: Install Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.8' | |
| - run: echo $JAVA_HOME | |
| - run: echo "JAVA11_HOME=${JAVA_HOME}" >> "$GITHUB_OUTPUT" | |
| id: export-java11 | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '8' | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Remove default github maven configuration | |
| # This step is a workaround to avoid a decryption issue of Beam's | |
| # net.linguica.gradle.maven.settings plugin and github's provided maven | |
| # settings.xml file | |
| run: rm ~/.m2/settings.xml || true | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v2 | |
| with: | |
| username: ${{ secrets.DOCKERHUB_USER }} | |
| password: ${{ secrets.DOCKERHUB_TOKEN }} | |
| - name: Push docker images | |
| run: ./gradlew :pushAllDockerImages -PisRelease -Pdocker-pull-licenses -Pprune-images -Pdocker-tag=${{ github.event.inputs.RELEASE }}rc${{ github.event.inputs.RC }} -Pjava11Home=${{steps.export-java11.outputs.JAVA11_HOME}} --no-daemon --no-parallel | |
| beam_site_pr: | |
| if: ${{github.event.inputs.CREATE_BEAM_SITE_PR == 'yes'}} | |
| # Note: if this ever changes to self-hosted, remove the "Remove default github maven configuration" step | |
| runs-on: ubuntu-latest | |
| env: | |
| RC_TAG: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" | |
| BRANCH_NAME: updates_release_${{ github.event.inputs.RELEASE }} | |
| BEAM_ROOT_DIR: ${{ github.workspace }}/beam | |
| SITE_ROOT_DIR: ${{ github.workspace }}/beam-site | |
| steps: | |
| - name: Checkout Beam Repo | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: "v${{ github.event.inputs.RELEASE }}-RC${{ github.event.inputs.RC }}" | |
| repository: apache/beam | |
| path: beam | |
| - name: Checkout Beam Site Repo | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: apache/beam-site | |
| path: beam-site | |
| token: ${{ github.event.inputs.BEAM_SITE_TOKEN }} | |
| ref: release-docs | |
| - name: Install Python 3.8 | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.8' | |
| - name: Install node | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '16' | |
| - name: Install Java 8 | |
| uses: actions/setup-java@v4 | |
| with: | |
| distribution: 'temurin' | |
| java-version: '8' | |
| - name: Remove default github maven configuration | |
| # This step is a workaround to avoid a decryption issue of Beam's | |
| # net.linguica.gradle.maven.settings plugin and github's provided maven | |
| # settings.xml file | |
| run: rm ~/.m2/settings.xml || true | |
| - name: Create documentation commit for website | |
| run: | | |
| echo "OK!" | |
| - name: Build Python Docs | |
| working-directory: beam/sdks/python | |
| run: | | |
| pip install --upgrade pip setuptools wheel | |
| pip install -U pip | |
| pip install tox | |
| # TODO(https://github.com/apache/beam/issues/20209): Don't hardcode py version in this file. | |
| tox -e py38-docs | |
| rm -rf target/docs/_build/.doctrees | |
| - name: Build Typescript Docs | |
| working-directory: beam/sdks/typescript | |
| run: | | |
| npm ci && npm run docs | |
| - name: Build Java Docs | |
| working-directory: beam | |
| run: | | |
| ./gradlew :sdks:java:javadoc:aggregateJavadoc -PisRelease --no-daemon --no-parallel | |
| - name: Add canonical link into javadocs | |
| uses: cicirello/javadoc-cleanup@v1 | |
| with: | |
| path-to-root: beam/sdks/java/javadoc/build/docs/javadoc | |
| base-url-path: https://beam.apache.org/releases/javadoc/current/ | |
| - name: Consolidate Release Docs to beam-site branch with symlinks | |
| working-directory: beam-site | |
| run: | | |
| git checkout -b $BRANCH_NAME release-docs | |
| echo "..........Copying generated javadoc into beam-site.........." | |
| cp -r ${BEAM_ROOT_DIR}/sdks/java/javadoc/build/docs/javadoc/ javadoc/${{ github.event.inputs.RELEASE }} | |
| # Update current symlink to point to the latest release | |
| unlink javadoc/current | |
| ln -s ${{ github.event.inputs.RELEASE }} javadoc/current | |
| echo "............Copying generated pydoc into beam-site.........." | |
| cp -r ${BEAM_ROOT_DIR}/sdks/python/target/docs/_build pydoc/${{ github.event.inputs.RELEASE }} | |
| # Update current symlink to point to the latest release | |
| unlink pydoc/current | |
| ln -s ${{ github.event.inputs.RELEASE }} pydoc/current | |
| echo "............Copying generated typedoc into beam-site.........." | |
| mkdir -p typedoc | |
| cp -r ${BEAM_ROOT_DIR}/sdks/typescript/docs typedoc/${{ github.event.inputs.RELEASE }} | |
| # Update current symlink to point to the latest release | |
| unlink typedoc/current | true | |
| ln -s ${{ github.event.inputs.RELEASE }} typedoc/current | |
| - name: Create commit on beam-site branch | |
| working-directory: beam | |
| run: | | |
| # Get the commit from the beam repo, not the beam-site repo. | |
| RELEASE_COMMIT=$(git rev-list -n 1 "tags/${RC_TAG}") | |
| cd ${SITE_ROOT_DIR} | |
| git config user.name $GITHUB_ACTOR | |
| git config user.email actions@"$RUNNER_NAME".local | |
| git add -A | |
| git commit -m "Update beam-site for release ${{ github.event.inputs.RELEASE }}." -m "Content generated from commit ${RELEASE_COMMIT}." | |
| git push -f --set-upstream origin $BRANCH_NAME | |
| - name: Create beam-site PR | |
| working-directory: beam-site | |
| env: | |
| GH_TOKEN: ${{ github.event.inputs.BEAM_SITE_TOKEN }} | |
| PR_TITLE: "Publish docs for ${{ github.event.inputs.RELEASE }} release" | |
| PR_BODY: "Content generated from https://github.com/apache/beam/tree/${{ env.RC_TAG }}." | |
| run: | | |
| gh pr create -t "$PR_TITLE" -b "$PR_BODY" --base release-docs |