From c017200e85ab2b0bc466e1314222fcbe879178d2 Mon Sep 17 00:00:00 2001 From: Keeley Hammond Date: Mon, 2 May 2022 09:50:42 -0700 Subject: [PATCH] build: improve CI speeds and reduce CI costs (#33904) (#33952) * build: improve CI speeds and reduce CI costs (#33904) * remove third_party/electron_node:overlapped-checker target isn't present in older versions * build: use original arch logic Co-authored-by: John Kleinschmidt Co-authored-by: Samuel Attard --- .circleci/config/base.yml | 216 +++++++++++++++++--------------- BUILD.gn | 4 + script/node-disabled-tests.json | 1 + 3 files changed, 119 insertions(+), 102 deletions(-) diff --git a/.circleci/config/base.yml b/.circleci/config/base.yml index b39a20ecc263a..50c57655f6308 100644 --- a/.circleci/config/base.yml +++ b/.circleci/config/base.yml @@ -336,6 +336,19 @@ step-setup-goma-for-build: &step-setup-goma-for-build echo 'export LOCAL_GOMA_DIR='`node -e "console.log(require('./src/utils/goma.js').dir)"` >> $BASH_ENV echo 'export GOMA_FALLBACK_ON_AUTH_FAILURE=true' >> $BASH_ENV cd .. + touch "${TMPDIR:=/tmp}"/.goma-ready + background: true + +step-wait-for-goma: &step-wait-for-goma + run: + name: Wait for Goma + command: | + until [ -f "${TMPDIR:=/tmp}"/.goma-ready ] + do + sleep 5 + done + echo "Goma ready" + no_output_timeout: 2m step-restore-brew-cache: &step-restore-brew-cache restore_cache: @@ -559,14 +572,6 @@ step-electron-build: &step-electron-build cp out/Default/.ninja_log out/electron_ninja_log node electron/script/check-symlinks.js -step-native-unittests-build: &step-native-unittests-build - run: - name: Build native test targets - no_output_timeout: 30m - command: | - cd src - ninja -C out/Default shell_browser_ui_unittests -j $NUMBER_OF_NINJA_PROCESSES - step-maybe-electron-dist-strip: &step-maybe-electron-dist-strip run: name: Strip electron binaries @@ -585,40 +590,6 @@ step-maybe-electron-dist-strip: &step-maybe-electron-dist-strip electron/script/add-debug-link.py --target-cpu="$target_cpu" --debug-dir=out/Default/debug fi -step-electron-dist-build: &step-electron-dist-build - run: - name: Build dist.zip - command: | - cd src - if [ "$SKIP_DIST_ZIP" != "1" ]; then - ninja -C out/Default electron:electron_dist_zip - if [ "$CHECK_DIST_MANIFEST" == "1" ]; then - if [ "`uname`" == "Darwin" ]; then - target_os=mac - target_cpu=x64 - if [ x"$MAS_BUILD" == x"true" ]; then - target_os=mac_mas - fi - if [ "$TARGET_ARCH" == "arm64" ]; then - target_cpu=arm64 - fi - elif [ "`uname`" == "Linux" ]; then - target_os=linux - if [ x"$TARGET_ARCH" == x ]; then - target_cpu=x64 - elif [ "$TARGET_ARCH" == "ia32" ]; then - target_cpu=x86 - else - target_cpu="$TARGET_ARCH" - fi - else - echo "Unknown system: `uname`" - exit 1 - fi - electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest - fi - fi - step-electron-chromedriver-build: &step-electron-chromedriver-build run: name: Build chromedriver.zip @@ -671,7 +642,6 @@ step-persist-data-for-tests: &step-persist-data-for-tests - src/out/Default/dist.zip - src/out/Default/mksnapshot.zip - src/out/Default/chromedriver.zip - - src/out/Default/shell_browser_ui_unittests - src/out/Default/gen/node_headers - src/out/ffmpeg/ffmpeg.zip - src/electron @@ -782,6 +752,7 @@ step-show-goma-stats: &step-show-goma-stats $LOCAL_GOMA_DIR/diagnose_goma_log.py true when: always + background: true step-mksnapshot-build: &step-mksnapshot-build run: @@ -1025,11 +996,15 @@ step-ts-compile: &step-ts-compile # List of all steps. steps-electron-gn-check: &steps-electron-gn-check steps: - - attach_workspace: - at: . + - *step-checkout-electron + - *step-depot-tools-get - *step-depot-tools-add-to-path - *step-setup-env-for-build - *step-setup-goma-for-build + - *step-generate-deps-hash + - *step-touch-sync-done + - maybe-restore-portaled-src-cache + - *step-wait-for-goma - *step-gn-gen-default - *step-gn-check @@ -1042,6 +1017,7 @@ steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-cha - *step-restore-brew-cache - *step-install-gnutar-on-mac - *step-get-more-space-on-mac + - *step-setup-goma-for-build - *step-generate-deps-hash - *step-touch-sync-done - maybe-restore-portaled-src-cache @@ -1061,7 +1037,7 @@ steps-electron-ts-compile-for-doc-change: &steps-electron-ts-compile-for-doc-cha - *step-depot-tools-add-to-path - *step-setup-env-for-build - - *step-setup-goma-for-build + - *step-wait-for-goma - *step-get-more-space-on-mac - *step-install-npm-deps-on-mac - *step-fix-sync @@ -1077,6 +1053,7 @@ steps-native-tests: &steps-native-tests - *step-depot-tools-add-to-path - *step-setup-env-for-build - *step-setup-goma-for-build + - *step-wait-for-goma - *step-gn-gen-default - run: @@ -1262,7 +1239,6 @@ commands: fi } mv_if_exist src/out/Default/dist.zip - mv_if_exist src/out/Default/shell_browser_ui_unittests mv_if_exist src/out/Default/gen/node_headers.tar.gz mv_if_exist src/out/Default/symbols.zip mv_if_exist src/out/Default/mksnapshot.zip @@ -1299,6 +1275,46 @@ commands: - *step-checkout-electron - *step-run-electron-only-hooks - *step-generate-deps-hash-cleanly + + step-electron-dist-build: + parameters: + additional-targets: + type: string + default: '' + steps: + - run: + name: Build dist.zip + command: | + cd src + if [ "$SKIP_DIST_ZIP" != "1" ]; then + ninja -C out/Default electron:electron_dist_zip << parameters.additional-targets >> + if [ "$CHECK_DIST_MANIFEST" == "1" ]; then + if [ "`uname`" == "Darwin" ]; then + target_os=mac + target_cpu=x64 + if [ x"$MAS_BUILD" == x"true" ]; then + target_os=mac_mas + fi + if [ "$TARGET_ARCH" == "arm64" ]; then + target_cpu=arm64 + fi + elif [ "`uname`" == "Linux" ]; then + target_os=linux + if [ x"$TARGET_ARCH" == x ]; then + target_cpu=x64 + elif [ "$TARGET_ARCH" == "ia32" ]; then + target_cpu=x86 + else + target_cpu="$TARGET_ARCH" + fi + else + echo "Unknown system: `uname`" + exit 1 + fi + electron/script/zip_manifests/check-zip-manifest.py out/Default/dist.zip electron/script/zip_manifests/dist_zip.$target_os.$target_cpu.manifest + fi + fi + electron-build: parameters: attach: @@ -1344,6 +1360,10 @@ commands: - *step-restore-brew-cache - *step-install-gnutar-on-mac - *step-save-brew-cache + - when: + condition: << parameters.build >> + steps: + - *step-setup-goma-for-build - when: condition: << parameters.checkout-and-assume-cache >> steps: @@ -1439,7 +1459,7 @@ commands: steps: - *step-depot-tools-add-to-path - *step-setup-env-for-build - - *step-setup-goma-for-build + - *step-wait-for-goma - *step-get-more-space-on-mac - *step-fix-sync - *step-delete-git-directories @@ -1452,13 +1472,8 @@ commands: - *step-gn-gen-default - *step-electron-build - *step-maybe-electron-dist-strip - - *step-electron-dist-build - - # Native test targets - - *step-native-unittests-build - - # Node.js headers - - *step-nodejs-headers-build + - step-electron-dist-build: + additional-targets: shell_browser_ui_unittests third_party/electron_node:headers electron:hunspell_dictionaries_zip - *step-show-goma-stats @@ -1476,13 +1491,14 @@ commands: - *step-ffmpeg-gn-gen - *step-ffmpeg-build - # hunspell - - *step-hunspell-build - # Save all data needed for a further tests run. - when: condition: << parameters.persist >> steps: + - *step-minimize-workspace-size-from-checkout + - run: | + rm -rf src/third_party/electron_node/deps/openssl + rm -rf src/third_party/electron_node/deps/v8 - *step-persist-data-for-tests - when: @@ -1548,6 +1564,7 @@ commands: - *step-fix-sync - *step-setup-env-for-build - *step-setup-goma-for-build + - *step-wait-for-goma - *step-gn-gen-default # Electron app @@ -1555,7 +1572,7 @@ commands: - *step-show-goma-stats - *step-maybe-generate-breakpad-symbols - *step-maybe-electron-dist-strip - - *step-electron-dist-build + - step-electron-dist-build - *step-maybe-zip-symbols # mksnapshot @@ -1598,20 +1615,10 @@ jobs: <<: *steps-electron-ts-compile-for-doc-change # Layer 1: Checkout. - linux-checkout-for-workspace: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' - steps: - - electron-build: - persist: false - build: false - checkout: true - persist-checkout: true - linux-make-src-cache: - executor: linux-docker + executor: + name: linux-docker + size: xlarge environment: <<: *env-linux-2xlarge GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' @@ -1662,22 +1669,10 @@ jobs: persist-checkout: true restore-src-cache: false - mac-checkout-for-workspace: - executor: linux-docker - environment: - <<: *env-linux-2xlarge - <<: *env-testing-build - <<: *env-macos-build - GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' - steps: - - electron-build: - persist: false - build: false - checkout: true - persist-checkout: true - mac-make-src-cache: - executor: linux-docker + executor: + name: linux-docker + size: xlarge environment: <<: *env-linux-2xlarge <<: *env-testing-build @@ -1702,7 +1697,8 @@ jobs: steps: - electron-build: persist: true - checkout: true + checkout: false + checkout-and-assume-cache: true use-out-cache: false linux-x64-testing-asan: @@ -1742,6 +1738,7 @@ jobs: environment: <<: *env-linux-medium <<: *env-testing-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' <<: *steps-electron-gn-check linux-x64-release: @@ -1838,7 +1835,8 @@ jobs: steps: - electron-build: persist: true - checkout: true + checkout: false + checkout-and-assume-cache: true use-out-cache: false linux-arm-release: @@ -1890,7 +1888,8 @@ jobs: steps: - electron-build: persist: true - checkout: true + checkout: false + checkout-and-assume-cache: true use-out-cache: false linux-arm64-testing-gn-check: @@ -1901,6 +1900,7 @@ jobs: <<: *env-linux-medium <<: *env-arm64 <<: *env-testing-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_arm=True --custom-var=checkout_arm64=True' <<: *steps-electron-gn-check linux-arm64-release: @@ -1962,6 +1962,7 @@ jobs: environment: <<: *env-machine-mac <<: *env-testing-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' <<: *steps-electron-gn-check osx-publish-x64-skip-checkout: @@ -2052,6 +2053,7 @@ jobs: <<: *env-machine-mac <<: *env-mas <<: *env-testing-build + GCLIENT_EXTRA_ARGS: '--custom-var=checkout_mac=True --custom-var=host_os=mac' <<: *steps-electron-gn-check mas-publish-x64-skip-checkout: @@ -2380,14 +2382,19 @@ workflows: - equal: [false, << pipeline.parameters.run-linux-publish >>] - equal: [true, << pipeline.parameters.run-build-linux >>] jobs: - - linux-checkout-for-workspace - linux-make-src-cache - - linux-x64-testing - - linux-x64-testing-asan - - linux-x64-testing-no-run-as-node + - linux-x64-testing: + requires: + - linux-make-src-cache + - linux-x64-testing-asan: + requires: + - linux-make-src-cache + - linux-x64-testing-no-run-as-node: + requires: + - linux-make-src-cache - linux-x64-testing-gn-check: requires: - - linux-checkout-for-workspace + - linux-make-src-cache - linux-x64-testing-tests: requires: - linux-x64-testing @@ -2400,7 +2407,9 @@ workflows: - linux-x64-testing-node: requires: - linux-x64-testing - - linux-ia32-testing + - linux-ia32-testing: + requires: + - linux-make-src-cache - linux-ia32-testing-tests: requires: - linux-ia32-testing @@ -2410,7 +2419,9 @@ workflows: - linux-ia32-testing-node: requires: - linux-ia32-testing - - linux-arm-testing + - linux-arm-testing: + requires: + - linux-make-src-cache - linux-arm-testing-tests: filters: branches: @@ -2418,7 +2429,9 @@ workflows: ignore: /pull\/[0-9]+/ requires: - linux-arm-testing - - linux-arm64-testing + - linux-arm64-testing: + requires: + - linux-make-src-cache - linux-arm64-testing-tests: filters: branches: @@ -2428,7 +2441,7 @@ workflows: - linux-arm64-testing - linux-arm64-testing-gn-check: requires: - - linux-checkout-for-workspace + - linux-make-src-cache build-mac: when: @@ -2437,14 +2450,13 @@ workflows: - equal: [false, << pipeline.parameters.run-linux-publish >>] - equal: [true, << pipeline.parameters.run-build-mac >>] jobs: - - mac-checkout-for-workspace - mac-make-src-cache - osx-testing-x64: requires: - mac-make-src-cache - osx-testing-x64-gn-check: requires: - - mac-checkout-for-workspace + - mac-make-src-cache - osx-testing-x64-tests: requires: - osx-testing-x64 @@ -2463,7 +2475,7 @@ workflows: - mac-make-src-cache - mas-testing-x64-gn-check: requires: - - mac-checkout-for-workspace + - mac-make-src-cache - mas-testing-x64-tests: requires: - mas-testing-x64 diff --git a/BUILD.gn b/BUILD.gn index db49d020f4b6b..fdd0419cf7587 100644 --- a/BUILD.gn +++ b/BUILD.gn @@ -1275,6 +1275,10 @@ if (is_mac) { if (!is_component_build && is_component_ffmpeg) { configs += [ "//build/config/gcc:rpath_for_built_shared_libraries" ] } + + if (is_linux) { + deps += [ "//sandbox/linux:chrome_sandbox" ] + } } } diff --git a/script/node-disabled-tests.json b/script/node-disabled-tests.json index 52128823432f0..41cb1a8794c01 100644 --- a/script/node-disabled-tests.json +++ b/script/node-disabled-tests.json @@ -27,6 +27,7 @@ "parallel/test-http2-clean-output", "parallel/test-https-agent-session-reuse", "parallel/test-https-options-boolean-check", + "parallel/test-icu-minimum-version", "parallel/test-inspector-multisession-ws", "parallel/test-inspector-port-zero-cluster", "parallel/test-inspector-tracing-domain",