From b41da150caa363d06f522427c87bd989464c9edd Mon Sep 17 00:00:00 2001 From: Shelley Vohr Date: Wed, 24 Apr 2024 13:29:19 -0400 Subject: [PATCH] build: add GitHub Actions testing pipeline for macOS (#41812) * build: test the Mac runner space * build: baby steps, super basic checkout * chore: simplify depot-tools * build: try a sync * build: fix path * build: pass different URL for sync * build: gclient sync with --verbose * debug: add logging for mtime cache * build: add psuedocode for Azure workflow, add workflow_dispatch * build: enable ssh access to runners * build: test azure upload without full sync * build: full sync and cache upload test * build (WIP): mock out download logic * build: remove sync, try disk space prune * build: lets try it again, lads * ci: add cache downloading wip * fix: correct json parsing * chore: use depshash and fix output syntax * chore: rework depshash state and delete more * refactor: split checkout to Linux runner * fix: depot_tools get on Linux * fix: add infra/3pp/tools/swift-format handling * chore: move to patch file in build as well * chore: misc cleanup of env steps * chore: add fix sync step * build: use tar file, strip .git dirs * build: set GIT_CACHE_PATH outside of src * ci: fix cache prefix, add check for ssh debug * fix: fix tar pack/unpack * build: add reclient support & ninja build step * chore: clean up reclient round 1 * fix: grab ELECTRON_RBE_JWT from secrets * chore: run tmate if debug logging enabled * chore: move ulimit setting * fix: missing GN gen env vars * fix: workaround packed-refs issue * chore: fixup PATH usage * chore: remove GIT_CACHE env var * fix: call pack refs manually * fix: add RBE_experimental_credentials_helper_args * chore: whoops we need electron_node/deps/v8 actually * fix: we need electron_node/deps/openssl too * build: load build-tools & xcode * build: add dist, chromedriver & mksnapshot * build: add test job, download artifacts * Cache non-generated build artifacts * chore: set download path explicitly * chore: load-xcode is called by e build * Add codesigning certificate step * Add Test Step * Try without codesigning just to get tests running * Add depot_tools to PATH in test * Persist src/buildtools/mac * chore: use script/yarn test directly * Build Node.js Headers for caching * Add initial matrix for mas build * Fix GN_EXTRA_ARGS generation syntax * Fix BUILD_TYPE interpolation * Use matrix in test correctly * feat: add publish config, alter release-build script * build: force trigger the first workflow * build: oops * Ensure env vars escaped * Move GN_EXTRA_ARGS to gn gen step * chore: rename yml to match convention * ci: fix helperPath calls in action configs * chore: try setting directly * fix: switch to mas config * chore: Testing -> Default * Re-enable darwin build * Use mas JSON config * refactor: extract artifact logic to bash scripts * refactor: build MAS after darwin * chore: move to script/actions * chore: use Node.js 20.11.x * Don't fail-fast in tests * build: tweak release script * build: remove macos-publish.yml * build: change action to trigger on publish workflow dispatch * chore: remove SSH debugging using action-tmate * build: do not disable CircleCI config * chore: switch away from cURL for AzureCLI * chore: move evm.mas.json to .github/workflows/config * chore: remove publish logic from initial build flow * build: add patchup check for non-release builds * build: pin build-tools to sha 2bb63e2 --------- Co-authored-by: Keeley Hammond Co-authored-by: Keeley Hammond --- .github/workflows/config/evm.mas.json | 26 ++ .github/workflows/config/gclient.diff | 14 + .github/workflows/macos-build.yml | 583 ++++++++++++++++++++++++++ script/actions/move-artifacts.sh | 40 ++ script/actions/restore-artifacts.sh | 22 + script/azure_cli_deb_install.sh | 91 ++++ 6 files changed, 776 insertions(+) create mode 100644 .github/workflows/config/evm.mas.json create mode 100644 .github/workflows/config/gclient.diff create mode 100644 .github/workflows/macos-build.yml create mode 100755 script/actions/move-artifacts.sh create mode 100755 script/actions/restore-artifacts.sh create mode 100755 script/azure_cli_deb_install.sh diff --git a/.github/workflows/config/evm.mas.json b/.github/workflows/config/evm.mas.json new file mode 100644 index 0000000000000..6bacfe100f63a --- /dev/null +++ b/.github/workflows/config/evm.mas.json @@ -0,0 +1,26 @@ +{ + "root": "/Users/runner/work/electron/electron/", + "remotes": { + "electron": { + "origin": "https://github.com/electron/electron.git" + } + }, + "gen": { + "args": [ + "import(\"//electron/build/args/testing.gn\")", + "use_remoteexec = true", + "target_cpu = \"arm64\"", + "is_mas_build = true" + ], + "out": "Default" + }, + "env": { + "CHROMIUM_BUILDTOOLS_PATH": "/Users/runner/work/electron/electron/src/buildtools", + "GIT_CACHE_PATH": "/Users/runner/work/electron/electron/.git-cache" + }, + "$schema": "file:///home/builduser/.electron_build_tools/evm-config.schema.json", + "configValidationLevel": "strict", + "reclient": "remote_exec", + "goma": "none", + "preserveXcode": 5 +} diff --git a/.github/workflows/config/gclient.diff b/.github/workflows/config/gclient.diff new file mode 100644 index 0000000000000..3748ed5eec05d --- /dev/null +++ b/.github/workflows/config/gclient.diff @@ -0,0 +1,14 @@ +diff --git a/gclient.py b/gclient.py +index 59e2b4c5197928bdba1ef69bdbe637d7dfe471c1..b4bae5e48c83c84bd867187afaf40eed16e69851 100755 +--- a/gclient.py ++++ b/gclient.py +@@ -739,7 +739,8 @@ class Dependency(gclient_utils.WorkItem, DependencySettings): + + if dep_type == 'cipd': + cipd_root = self.GetCipdRoot() +- for package in dep_value.get('packages', []): ++ packages = dep_value.get('packages', []) ++ for package in (x for x in packages if "infra/3pp/tools/swift-format" not in x.get('package')): + deps_to_add.append( + CipdDependency(parent=self, + name=name, diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml new file mode 100644 index 0000000000000..e7304a278b158 --- /dev/null +++ b/.github/workflows/macos-build.yml @@ -0,0 +1,583 @@ +name: Build MacOS + +on: + workflow_dispatch: + +env: + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac --custom-var=host_cpu=arm64' + IS_RELEASE: false + # GENERATE_SYMBOLS: true only on release builds + GENERATE_SYMBOLS: false + AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }} + AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} + AZURE_STORAGE_CONTAINER_NAME: ${{ secrets.AZURE_STORAGE_CONTAINER_NAME }} + ELECTRON_RBE_JWT: ${{ secrets.ELECTRON_RBE_JWT }} + # TODO: this should be set to the correct GN_CONFIG for the build type. + GN_CONFIG: //electron/build/args/testing.gn + # Disable pre-compiled headers to reduce out size - only useful for rebuilds + GN_BUILDFLAG_ARGS: 'enable_precompiled_headers = false' + +jobs: + checkout: + runs-on: LargeLinuxRunner + steps: + - name: Checkout Electron + uses: actions/checkout@v4 + with: + path: src/electron + - name: Install Azure CLI + run: ./script/azure_cli_deb_install.sh + - name: Set GIT_CACHE_PATH to make gclient to use the cache + run: | + echo "GIT_CACHE_PATH=$(pwd)/git-cache" >> $GITHUB_ENV + - name: Setup Node.js/npm + uses: actions/setup-node@v3 + with: + node-version: 20.11.x + cache: yarn + cache-dependency-path: src/electron/yarn.lock + - name: Install Dependencies + run: | + cd src/electron + node script/yarn install + - name: Get Depot Tools + timeout-minutes: 5 + run: | + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + if [ "`uname`" == "Darwin" ]; then + # remove ninjalog_uploader_wrapper.py from autoninja since we don't use it and it causes problems + sed -i '' '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja + else + sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja + # Remove swift-format dep from cipd on macOS until we send a patch upstream. + cd depot_tools + git apply --3way ../src/electron/.github/workflows/config/gclient.diff + fi + # Ensure depot_tools does not update. + test -d depot_tools && cd depot_tools + touch .disable_auto_update + - name: Add Depot Tools to PATH + run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH + - name: Generate DEPS Hash + run: | + node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target + echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV + - name: Check If Cache Exists + id: check-cache + run: | + exists_json=$(az storage blob exists \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --account-key $AZURE_STORAGE_KEY \ + --container-name $AZURE_STORAGE_CONTAINER_NAME \ + --name $DEPSHASH) + + cache_exists=$(echo $exists_json | jq -r '.exists') + echo "cache_exists=$cache_exists" >> $GITHUB_OUTPUT + + if (test "$cache_exists" = "true"); then + echo "Cache Exists for $DEPSHASH" + else + echo "Cache Does Not Exist for $DEPSHASH" + fi + - name: Gclient Sync + # If there is no existing src cache, we need to do a full gclient sync. + # TODO(codebytere): Add stale patch handling for non-release builds. + if: steps.check-cache.outputs.cache_exists == 'false' + run: | + gclient config \ + --name "src/electron" \ + --unmanaged \ + ${GCLIENT_EXTRA_ARGS} \ + "$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" + + ELECTRON_USE_THREE_WAY_MERGE_FOR_PATCHES=1 gclient sync --with_branch_heads --with_tags -vvvvv + if [ "$IS_RELEASE" != "true" ]; then + # Re-export all the patches to check if there were changes. + python3 src/electron/script/export_all_patches.py src/electron/patches/config.json + cd src/electron + git update-index --refresh || true + if ! git diff-index --quiet HEAD --; then + # There are changes to the patches. Make a git commit with the updated patches + git add patches + GIT_COMMITTER_NAME="PatchUp" GIT_COMMITTER_EMAIL="73610968+patchup[bot]@users.noreply.github.com" git commit -m "chore: update patches" --author="PatchUp <73610968+patchup[bot]@users.noreply.github.com>" + # Export it + mkdir -p ../../patches + git format-patch -1 --stdout --keep-subject --no-stat --full-index > ../../patches/update-patches.patch + if (node ./script/push-patch.js 2> /dev/null > /dev/null); then + echo + echo "======================================================================" + echo "Changes to the patches when applying, we have auto-pushed the diff to the current branch" + echo "A new CI job will kick off shortly" + echo "======================================================================" + exit 1 + else + echo + echo "======================================================================" + echo "There were changes to the patches when applying." + echo "Check the CI artifacts for a patch you can apply to fix it." + echo "======================================================================" + exit 1 + fi + fi + fi + + # delete all .git directories under src/ except for + # third_party/angle/ and third_party/dawn/ because of build time generation of files + # gen/angle/commit.h depends on third_party/angle/.git/HEAD + # https://chromium-review.googlesource.com/c/angle/angle/+/2074924 + # and dawn/common/Version_autogen.h depends on third_party/dawn/.git/HEAD + # https://dawn-review.googlesource.com/c/dawn/+/83901 + # TODO: maybe better to always leave out */.git/HEAD file for all targets ? + - name: Delete .git directories under src to free space + if: steps.check-cache.outputs.cache_exists == 'false' + run: | + cd src + ( find . -type d -name ".git" -not -path "./third_party/angle/*" -not -path "./third_party/dawn/*" -not -path "./electron/*" ) | xargs rm -rf + - name: Minimize Cache Size for Upload + if: steps.check-cache.outputs.cache_exists == 'false' + run: | + rm -rf src/android_webview + rm -rf src/ios/chrome + rm -rf src/third_party/blink/web_tests + rm -rf src/third_party/blink/perf_tests + rm -rf src/chrome/test/data/xr/webvr_info + rm -rf src/third_party/angle/third_party/VK-GL-CTS/src + rm -rf src/third_party/swift-toolchain + rm -rf src/third_party/swiftshader/tests/regres/testlists + rm -rf src/electron + - name: Compress Src Directory + if: steps.check-cache.outputs.cache_exists == 'false' + run: | + echo "Uncompressed src size: $(du -sh src | cut -f1 -d' ')" + tar -cvf $DEPSHASH.tar src + echo "Compressed src to $(du -sh $DEPSHASH.tar | cut -f1 -d' ')" + - name: Upload Compressed Src Cache to Azure + if: steps.check-cache.outputs.cache_exists == 'false' + run: | + az storage blob upload \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --account-key $AZURE_STORAGE_KEY \ + --container-name $AZURE_STORAGE_CONTAINER_NAME \ + --file $DEPSHASH.tar \ + --name $DEPSHASH \ + --debug + build: + runs-on: macos-13-xlarge + needs: checkout + steps: + - name: Load Build Tools + run: | + yarn add git://github.com/electron/build-tools.git#2bb63e2e7877491b52f972532b52adc979a6ec2f + e init --root=$(pwd) --out=Default testing + - name: Checkout Electron + uses: actions/checkout@v4 + with: + path: src/electron + - name: Install Azure CLI + run: | + brew update && brew install azure-cli + - name: Setup Node.js/npm + uses: actions/setup-node@v3 + with: + node-version: 20.11.x + cache: yarn + cache-dependency-path: src/electron/yarn.lock + - name: Install Dependencies + run: | + cd src/electron + node script/yarn install + - name: Get Depot Tools + timeout-minutes: 5 + run: | + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + if [ "`uname`" == "Darwin" ]; then + # remove ninjalog_uploader_wrapper.py from autoninja since we don't use it and it causes problems + sed -i '' '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja + else + sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja + # Remove swift-format dep from cipd on macOS until we send a patch upstream. + cd depot_tools + git apply --3way ../src/electron/.github/workflows/config/gclient.diff + fi + # Ensure depot_tools does not update. + test -d depot_tools && cd depot_tools + touch .disable_auto_update + - name: Add Depot Tools to PATH + run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH + - name: Generate DEPS Hash + run: | + node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target + echo "DEPSHASH=v1-src-cache-$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV + - name: Download Src Cache + # The cache will always exist here as a result of the checkout job + # Either it was uploaded to Azure in the checkout job for this commit + # or it was uploaded in the checkout job for a previous commit. + run: | + az storage blob download \ + --account-name $AZURE_STORAGE_ACCOUNT \ + --account-key $AZURE_STORAGE_KEY \ + --container-name $AZURE_STORAGE_CONTAINER_NAME \ + --name $DEPSHASH \ + --file $DEPSHASH.tar \ + - name: Unzip and Ensure Src Cache + run: | + echo "Downloaded cache is $(du -sh $DEPSHASH.tar | cut -f1)" + mkdir temp-cache + tar -xvf $DEPSHASH.tar -C temp-cache + echo "Unzipped cache is $(du -sh temp-cache/src | cut -f1)" + + if [ -d "temp-cache/src" ]; then + echo "Relocating Cache" + rm -rf src + mv temp-cache/src src + + echo "Deleting zip file" + rm -rf $DEPSHASH.tar + fi + + if [ ! -d "src/third_party/blink" ]; then + echo "Cache was not correctly restored - exiting" + exit 1 + fi + + echo "Wiping Electron Directory" + rm -rf src/electron + - name: Checkout Electron + uses: actions/checkout@v4 + with: + path: src/electron + - name: Run Electron Only Hooks + run: | + echo "Running Electron Only Hooks" + gclient runhooks --spec="solutions=[{'name':'src/electron','url':None,'deps_file':'DEPS','custom_vars':{'process_deps':False},'managed':False}]" + - name: Regenerate DEPS Hash + run: | + (cd src/electron && git checkout .) && node src/electron/script/generate-deps-hash.js && cat src/electron/.depshash-target + echo "DEPSHASH=$(shasum src/electron/.depshash | cut -f1 -d' ')" >> $GITHUB_ENV + - name: Add CHROMIUM_BUILDTOOLS_PATH to env + run: echo "CHROMIUM_BUILDTOOLS_PATH=$(pwd)/src/buildtools" >> $GITHUB_ENV + - name: Fix Sync + # This step is required to correct for differences between "gclient sync" + # on Linux and the expected state on macOS. This requires: + # 1. Fixing Clang Install (wrong binary) + # 2. Fixing esbuild (wrong binary) + # 3. Fixing rustc (wrong binary) + # 4. Fixing gn (wrong binary) + # 5. Fix reclient (wrong binary) + # 6. Fixing dsymutil (wrong binary) + # 7. Ensuring we are using the correct ninja and adding it to PATH + # 8. Fixing angle (wrong remote) + run : | + SEDOPTION="-i ''" + rm -rf src/third_party/llvm-build + python3 src/tools/clang/scripts/update.py + + echo 'infra/3pp/tools/esbuild/${platform}' `gclient getdep --deps-file=src/third_party/devtools-frontend/src/DEPS -r 'third_party/esbuild:infra/3pp/tools/esbuild/${platform}'` > esbuild_ensure_file + # Remove extra output from calling gclient getdep which always calls update_depot_tools + sed -i '' "s/Updating depot_tools... //g" esbuild_ensure_file + cipd ensure --root src/third_party/devtools-frontend/src/third_party/esbuild -ensure-file esbuild_ensure_file + + rm -rf src/third_party/rust-toolchain + python3 src/tools/rust/update_rust.py + + # Prevent calling gclient getdep which always calls update_depot_tools + echo 'gn/gn/mac-${arch}' `gclient getdep --deps-file=src/DEPS -r 'src/buildtools/mac:gn/gn/mac-${arch}'` > gn_ensure_file + sed -i '' "s/Updating depot_tools... //g" gn_ensure_file + cipd ensure --root src/buildtools/mac -ensure-file gn_ensure_file + + # Prevent calling gclient getdep which always calls update_depot_tools + echo 'infra/rbe/client/${platform}' `gclient getdep --deps-file=src/DEPS -r 'src/buildtools/reclient:infra/rbe/client/${platform}'` > gn_ensure_file + sed -i '' "s/Updating depot_tools... //g" gn_ensure_file + cipd ensure --root src/buildtools/reclient -ensure-file gn_ensure_file + python3 src/buildtools/reclient_cfgs/configure_reclient_cfgs.py --rbe_instance "projects/rbe-chrome-untrusted/instances/default_instance" --reproxy_cfg_template reproxy.cfg.template --rewrapper_cfg_project "" --skip_remoteexec_cfg_fetch + + DSYM_SHA_FILE=src/tools/clang/dsymutil/bin/dsymutil.arm64.sha1 + python3 src/third_party/depot_tools/download_from_google_storage.py --no_resume --no_auth --bucket chromium-browser-clang -s $DSYM_SHA_FILE -o src/tools/clang/dsymutil/bin/dsymutil + + echo 'infra/3pp/tools/ninja/${platform}' `gclient getdep --deps-file=src/DEPS -r 'src/third_party/ninja:infra/3pp/tools/ninja/${platform}'` > ninja_ensure_file + sed $SEDOPTION "s/Updating depot_tools... //g" ninja_ensure_file + cipd ensure --root src/third_party/ninja -ensure-file ninja_ensure_file + + echo "$(pwd)/src/third_party/ninja" >> $GITHUB_PATH + + cd src/third_party/angle + rm -f .git/objects/info/alternates + git remote set-url origin https://chromium.googlesource.com/angle/angle.git + cp .git/config .git/config.backup + git remote remove origin + mv .git/config.backup .git/config + git fetch + - name: Install build-tools & Setup RBE + run: | + echo "NUMBER_OF_NINJA_PROCESSES=200" >> $GITHUB_ENV + cd ~/.electron_build_tools + npx yarn --ignore-engines + # Pull down credential helper and print status + node -e "require('./src/utils/reclient.js').downloadAndPrepare({})" + HELPER=$(node -p "require('./src/utils/reclient.js').helperPath({})") + $HELPER login + echo 'RBE_service='`node -e "console.log(require('./src/utils/reclient.js').serviceAddress)"` >> $GITHUB_ENV + echo 'RBE_experimental_credentials_helper='`node -e "console.log(require('./src/utils/reclient.js').helperPath({}))"` >> $GITHUB_ENV + echo 'RBE_experimental_credentials_helper_args=print' >> $GITHUB_ENV + - name: Build Electron (darwin) + run: | + cd src/electron + # TODO(codebytere): remove this once we figure out why .git/packed-refs is initially missing + git pack-refs + cd .. + + ulimit -n 10000 + sudo launchctl limit maxfiles 65536 200000 + NINJA_SUMMARIZE_BUILD=1 e build -j $NUMBER_OF_NINJA_PROCESSES + cp out/Default/.ninja_log out/electron_ninja_log + node electron/script/check-symlinks.js + - name: Build Electron dist.zip (darwin) + run: | + cd src + e build electron:electron_dist_zip $ADDITIONAL_TARGETS -j $NUMBER_OF_NINJA_PROCESSES + if [ "$CHECK_DIST_MANIFEST" == "1" ]; then + target_os=mac + target_cpu=arm64 + electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest + fi + - name: Build Mksnapshot (darwin) + run: | + cd src + e build electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES + gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + # Remove unused args from mksnapshot_args + SEDOPTION="-i ''" + sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args + sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args + sed $SEDOPTION '/The gn arg use_goma=true .*/d' out/Default/mksnapshot_args + if [ "$SKIP_DIST_ZIP" != "1" ]; then + e build electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S) + fi + - name: Build Chromedriver (darwin) + run: | + cd src + e build electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES + e build electron:electron_chromedriver_zip + - name: Build Node Headers (darwin) + run: | + cd src + e build electron:node_headers + - name: Generate & Zip Symbols (darwin) + run: | + if [ "$GENERATE_SYMBOLS" == "true" ]; then + e build electron:electron_symbols + fi + cd src + export BUILD_PATH="$(pwd)/out/Default" + e build electron:licenses + e build electron:electron_version_file + electron/script/zip-symbols.py -b $BUILD_PATH + # TODO(vertedinde): handle creating ffmpeg and hunspell for release builds + # The current generated_artifacts_<< artifact.key >> name was taken from CircleCI + # tp ensure we don't break anything, but we may be able to improve that. + - name: Move all Generated Artifacts to Upload Folder + run: ./src/electron/script/actions/move-artifacts.sh + - name: Upload Generated Artifacts + uses: actions/upload-artifact@v4 + with: + name: generated_artifacts_darwin + path: ./generated_artifacts_darwin + - name: Persist Build Artifacts + uses: actions/cache/save@v4 + with: + path: | + src/out/Default/gen/node_headers + src/out/Default/overlapped-checker + src/electron + src/third_party/electron_node + src/third_party/nan + src/cross-arch-snapshots + src/third_party/llvm-build + src/build/linux + src/buildtools/mac + src/buildtools/third_party/libc++ + src/buildtools/third_party/libc++abi + src/third_party/libc++ + src/third_party/libc++abi + src/out/Default/obj/buildtools/third_party + src/v8/tools/builtins-pgo + key: ${{ runner.os }}-build-artifacts-darwin-${{ github.sha }} + - name: Create MAS Config + run: | + mv src/electron/.github/workflows/config/evm.mas.json $HOME/.electron_build_tools/configs/evm.mas.json + echo "MAS_BUILD=true" >> $GITHUB_ENV + e use mas + - name: Build Electron (mas) + run: | + rm -rf "src/out/Default/Electron Framework.framework" + rm -rf src/out/Default/Electron*.app + + cd src/electron + # TODO(codebytere): remove this once we figure out why .git/packed-refs is initially missing + git pack-refs + cd .. + + ulimit -n 10000 + sudo launchctl limit maxfiles 65536 200000 + NINJA_SUMMARIZE_BUILD=1 e build -j $NUMBER_OF_NINJA_PROCESSES + cp out/Default/.ninja_log out/electron_ninja_log + node electron/script/check-symlinks.js + - name: Build Electron dist.zip (mas) + run: | + cd src + e build electron:electron_dist_zip $ADDITIONAL_TARGETS -j $NUMBER_OF_NINJA_PROCESSES + if [ "$CHECK_DIST_MANIFEST" == "1" ]; then + target_os=mac_mas + target_cpu=arm64 + electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest + fi + - name: Build Mksnapshot (mas) + run: | + cd src + e build electron:electron_mksnapshot -j $NUMBER_OF_NINJA_PROCESSES + gn desc out/Default v8:run_mksnapshot_default args > out/Default/mksnapshot_args + # Remove unused args from mksnapshot_args + SEDOPTION="-i ''" + sed $SEDOPTION '/.*builtins-pgo/d' out/Default/mksnapshot_args + sed $SEDOPTION '/--turbo-profiling-input/d' out/Default/mksnapshot_args + sed $SEDOPTION '/The gn arg use_goma=true .*/d' out/Default/mksnapshot_args + if [ "$SKIP_DIST_ZIP" != "1" ]; then + e build electron:electron_mksnapshot_zip -j $NUMBER_OF_NINJA_PROCESSES + (cd out/Default; zip mksnapshot.zip mksnapshot_args gen/v8/embedded.S) + fi + - name: Build Chromedriver (mas) + run: | + cd src + e build electron:electron_chromedriver -j $NUMBER_OF_NINJA_PROCESSES + e build electron:electron_chromedriver_zip + - name: Build Node Headers + run: | + cd src + e build electron:node_headers + - name: Generate & Zip Symbols (mas) + run: | + if [ "$GENERATE_SYMBOLS" == "true" ]; then + e build electron:electron_symbols + fi + cd src + export BUILD_PATH="$(pwd)/out/Default" + e build electron:licenses + e build electron:electron_version_file + electron/script/zip-symbols.py -b $BUILD_PATH + - name: Move all Generated Artifacts to Upload Folder (mas) + run: ./src/electron/script/actions/move-artifacts.sh + - name: Upload Generated Artifacts + uses: actions/upload-artifact@v4 + with: + name: generated_artifacts_mas + path: ./generated_artifacts_mas + - name: Persist Build Artifacts + uses: actions/cache/save@v4 + with: + path: | + src/out/Default/gen/node_headers + src/out/Default/overlapped-checker + src/electron + src/third_party/electron_node + src/third_party/nan + src/cross-arch-snapshots + src/third_party/llvm-build + src/build/linux + src/buildtools/mac + src/buildtools/third_party/libc++ + src/buildtools/third_party/libc++abi + src/third_party/libc++ + src/third_party/libc++abi + src/out/Default/obj/buildtools/third_party + src/v8/tools/builtins-pgo + key: ${{ runner.os }}-build-artifacts-mas-${{ github.sha }} + test: + runs-on: macos-13-xlarge + needs: build + strategy: + fail-fast: false + matrix: + build-type: [darwin, mas] + env: + BUILD_TYPE: ${{ matrix.build-type }} + steps: + - name: Load Build Tools + run: | + yarn add git://github.com/electron/build-tools.git#2bb63e2e7877491b52f972532b52adc979a6ec2f + e init --root=$(pwd) --out=Default testing + - name: Checkout Electron + uses: actions/checkout@v4 + with: + path: src/electron + - name: Setup Node.js/npm + uses: actions/setup-node@v3 + with: + node-version: 20.11.x + cache: yarn + cache-dependency-path: src/electron/yarn.lock + - name: Install Dependencies + run: | + cd src/electron + node script/yarn install + - name: Get Depot Tools + timeout-minutes: 5 + run: | + git clone --depth=1 https://chromium.googlesource.com/chromium/tools/depot_tools.git + if [ "`uname`" == "Darwin" ]; then + # remove ninjalog_uploader_wrapper.py from autoninja since we don't use it and it causes problems + sed -i '' '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja + else + sed -i '/ninjalog_uploader_wrapper.py/d' ./depot_tools/autoninja + # Remove swift-format dep from cipd on macOS until we send a patch upstream. + cd depot_tools + git apply --3way ../src/electron/.github/workflows/config/gclient.diff + fi + # Ensure depot_tools does not update. + test -d depot_tools && cd depot_tools + touch .disable_auto_update + - name: Add Depot Tools to PATH + run: echo "$(pwd)/depot_tools" >> $GITHUB_PATH + - name: Download Generated Artifacts + uses: actions/download-artifact@v4 + with: + name: generated_artifacts_${{ matrix.build-type }} + path: ./generated_artifacts_${{ matrix.build-type }} + - name: Restore Persisted Build Artifacts + uses: actions/cache/restore@v4 + with: + path: | + src/out/Default/gen/node_headers + src/out/Default/overlapped-checker + src/electron + src/third_party/electron_node + src/third_party/nan + src/cross-arch-snapshots + src/third_party/llvm-build + src/build/linux + src/buildtools/mac + src/buildtools/third_party/libc++ + src/buildtools/third_party/libc++abi + src/third_party/libc++ + src/third_party/libc++abi + src/out/Default/obj/buildtools/third_party + src/v8/tools/builtins-pgo + key: ${{ runner.os }}-build-artifacts-${{ matrix.build-type }}-${{ github.sha }} + - name: Restore Generated Artifacts + run: ./src/electron/script/actions/restore-artifacts.sh + - name: Unzip Dist, Mksnapshot & Chromedriver + run: | + cd src/out/Default + unzip -:o dist.zip + unzip -:o chromedriver.zip + unzip -:o mksnapshot.zip + - name: Run Electron Tests + env: + MOCHA_REPORTER: mocha-multi-reporters + ELECTRON_TEST_RESULTS_DIR: junit + MOCHA_MULTI_REPORTERS: mocha-junit-reporter, tap + ELECTRON_DISABLE_SECURITY_WARNINGS: 1 + ELECTRON_SKIP_NATIVE_MODULE_TESTS: true + run: | + cd src/electron + node script/yarn test --runners=main --trace-uncaught --enable-logging diff --git a/script/actions/move-artifacts.sh b/script/actions/move-artifacts.sh new file mode 100755 index 0000000000000..5502e4acf3d98 --- /dev/null +++ b/script/actions/move-artifacts.sh @@ -0,0 +1,40 @@ +#!/bin/sh + +set -eo pipefail + +if [ -z "$MAS_BUILD" ]; then + BUILD_TYPE="darwin" +else + BUILD_TYPE="mas" +fi + +rm -rf generated_artifacts_${BUILD_TYPE} +mkdir generated_artifacts_${BUILD_TYPE} + +mv_if_exist() { + if [ -f "$1" ] || [ -d "$1" ]; then + echo Storing $1 + mv $1 generated_artifacts_${BUILD_TYPE} + else + echo Skipping $1 - It is not present on disk + fi +} +cp_if_exist() { + if [ -f "$1" ] || [ -d "$1" ]; then + echo Storing $1 + cp $1 generated_artifacts_${BUILD_TYPE} + else + echo Skipping $1 - It is not present on disk + fi +} + +mv_if_exist src/out/Default/dist.zip +mv_if_exist src/out/Default/gen/node_headers.tar.gz +mv_if_exist src/out/Default/symbols.zip +mv_if_exist src/out/Default/mksnapshot.zip +mv_if_exist src/out/Default/chromedriver.zip +mv_if_exist src/out/ffmpeg/ffmpeg.zip +mv_if_exist src/out/Default/hunspell_dictionaries.zip +mv_if_exist src/cross-arch-snapshots +cp_if_exist src/out/electron_ninja_log +cp_if_exist src/out/Default/.ninja_log diff --git a/script/actions/restore-artifacts.sh b/script/actions/restore-artifacts.sh new file mode 100755 index 0000000000000..d873dfd87b7dc --- /dev/null +++ b/script/actions/restore-artifacts.sh @@ -0,0 +1,22 @@ +#!/bin/sh + +set -eo pipefail + +mv_if_exist() { + if [ -f "generated_artifacts_${BUILD_TYPE}/$1" ] || [ -d "generated_artifacts_${BUILD_TYPE}/$1" ]; then + echo Restoring $1 to $2 + mkdir -p $2 + mv generated_artifacts_${BUILD_TYPE}/$1 $2 + else + echo Skipping $1 - It is not present on disk + fi +} + +mv_if_exist dist.zip src/out/Default +mv_if_exist node_headers.tar.gz src/out/Default/gen +mv_if_exist symbols.zip src/out/Default +mv_if_exist mksnapshot.zip src/out/Default +mv_if_exist chromedriver.zip src/out/Default +mv_if_exist ffmpeg.zip src/out/ffmpeg +mv_if_exist hunspell_dictionaries.zip src/out/Default +mv_if_exist cross-arch-snapshots src \ No newline at end of file diff --git a/script/azure_cli_deb_install.sh b/script/azure_cli_deb_install.sh new file mode 100755 index 0000000000000..b10ce2cc983dd --- /dev/null +++ b/script/azure_cli_deb_install.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash + +####################################################################################################################### +# This script does three fundamental things: # +# 1. Add Microsoft's GPG Key has a trusted source of apt packages. # +# 2. Add Microsoft's repositories as a source for apt packages. # +# 3. Installs the Azure CLI from those repositories. # +# Given the nature of this script, it must be executed with elevated privileges, i.e. with `sudo`. # +# # +# Copied from https://azurecliprod.blob.core.windows.net/$root/deb_install.sh # +####################################################################################################################### + +set -e + +if [[ $# -ge 1 && $1 == "-y" ]]; then + global_consent=0 +else + global_consent=1 +fi + +function assert_consent { + if [[ $2 -eq 0 ]]; then + return 0 + fi + + echo -n "$1 [Y/n] " + read consent + if [[ ! "${consent}" == "y" && ! "${consent}" == "Y" && ! "${consent}" == "" ]]; then + echo "'${consent}'" + exit 1 + fi +} + +global_consent=0 # Artificially giving global consent after review-feedback. Remove this line to enable interactive mode + +setup() { + + assert_consent "Add packages necessary to modify your apt-package sources?" ${global_consent} + set -v + export DEBIAN_FRONTEND=noninteractive + apt-get update + apt-get install -y apt-transport-https lsb-release gnupg curl + set +v + + assert_consent "Add Microsoft as a trusted package signer?" ${global_consent} + set -v + curl -sL https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > /etc/apt/trusted.gpg.d/microsoft.gpg + set +v + + assert_consent "Add the Azure CLI Repository to your apt sources?" ${global_consent} + set -v + # Use env var DIST_CODE for the package dist name if provided + if [[ -z $DIST_CODE ]]; then + CLI_REPO=$(lsb_release -cs) + shopt -s nocasematch + ERROR_MSG="Unable to find a package for your system. Please check if an existing package in https://packages.microsoft.com/repos/azure-cli/dists/ can be used in your system and install with the dist name: 'curl -sL https://aka.ms/InstallAzureCLIDeb | sudo DIST_CODE= bash'" + if [[ ! $(curl -sL https://packages.microsoft.com/repos/azure-cli/dists/) =~ $CLI_REPO ]]; then + DIST=$(lsb_release -is) + if [[ $DIST =~ "Ubuntu" ]]; then + CLI_REPO="jammy" + elif [[ $DIST =~ "Debian" ]]; then + CLI_REPO="bookworm" + elif [[ $DIST =~ "LinuxMint" ]]; then + CLI_REPO=$(cat /etc/os-release | grep -Po 'UBUNTU_CODENAME=\K.*') || true + if [[ -z $CLI_REPO ]]; then + echo $ERROR_MSG + exit 1 + fi + else + echo $ERROR_MSG + exit 1 + fi + fi + else + CLI_REPO=$DIST_CODE + if [[ ! $(curl -sL https://packages.microsoft.com/repos/azure-cli/dists/) =~ $CLI_REPO ]]; then + echo "Unable to find an azure-cli package with DIST_CODE=$CLI_REPO in https://packages.microsoft.com/repos/azure-cli/dists/." + exit 1 + fi + fi + echo "deb [arch=$(dpkg --print-architecture)] https://packages.microsoft.com/repos/azure-cli/ ${CLI_REPO} main" \ + > /etc/apt/sources.list.d/azure-cli.list + apt-get update + set +v + + assert_consent "Install the Azure CLI?" ${global_consent} + apt-get install -y azure-cli + +} + +setup # ensure the whole file is downloaded before executing