From 1eb3b484353827684a3a732ad353363f48805e2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Thu, 29 Sep 2022 17:08:08 +0200 Subject: [PATCH 01/10] Run test262 again --- .github/workflows/ci.yml | 52 ++++++++++++++++++++++++++++++++++++++++ Makefile | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index bf27423ac4cd..009d439584d5 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -484,3 +484,55 @@ jobs: run: yarn up eslint@7.5.0 - name: Run babel/eslint tests run: node ./node_modules/.bin/jest eslint + + test262: + name: test262 + runs-on: ubuntu-latest + needs: build + steps: + - name: Checkout Babel + uses: actions/checkout@v3 + - name: Use Node.js latest + uses: actions/setup-node@v3 + with: + node-version: "*" + cache: "yarn" + - name: Install + run: yarn install + - uses: actions/download-artifact@v3 + with: + name: babel-artifact + - name: Checkout test runner + uses: actions/checkout@v3 + with: + repository: babel/babel-test262-runner + path: babel-test262-runner + - name: Install test runner + run: | + cd babel-test262-runner + npm ci + node lib/download-node + - name: Download test262 + run: make bootstrap-test262 + - name: Run test262 + run: | + cd babel-test262-runner + node lib/run-tests I_AM_SURE | tee ~/test262.tap + env: + BABEL_PATH: ".." + TEST262_PATH: ../build/test262 + - name: Create artifact with report results + uses: actions/upload-artifact@v3 + with: + name: test262-result + path: ~/test262.tap + - name: Download previous results + run: | + cd babel-test262-runner + node lib/download-main-artifact/index.mjs ~/test262-main.tap + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Compare previous main branch & current results + run: | + cd babel-test262-runner + node lib/compare-results ~/test262-main.tap ~/test262.tap | tee ~/diff.tap diff --git a/Makefile b/Makefile index e72c5c2aca02..8c7cca2a64a2 100644 --- a/Makefile +++ b/Makefile @@ -178,7 +178,7 @@ bootstrap-test262: rm -rf build/test262 mkdir -p build git clone --filter=blob:none --sparse --single-branch --shallow-since=2021-05-01 https://github.com/tc39/test262.git build/test262 - cd build/test262 && git sparse-checkout set "test" + cd build/test262 && git sparse-checkout set "test" "harness" cd build/test262 && git checkout -q $(TEST262_COMMIT) test-test262: From 9c0e6ce9c734957fda527d2ad93e72065dd6ab52 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Sat, 1 Oct 2022 10:49:12 +0200 Subject: [PATCH 02/10] More threads --- .github/workflows/ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 009d439584d5..830330a5051c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -521,6 +521,7 @@ jobs: env: BABEL_PATH: ".." TEST262_PATH: ../build/test262 + THREADS: 4 - name: Create artifact with report results uses: actions/upload-artifact@v3 with: From 6c3347c9a89005ad425107185bb080fd10471992 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Sat, 1 Oct 2022 18:48:30 +0200 Subject: [PATCH 03/10] Run in parallel jobs --- .github/workflows/ci.yml | 72 ++++++++++++++++++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 830330a5051c..c744c8d958ee 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -485,10 +485,41 @@ jobs: - name: Run babel/eslint tests run: node ./node_modules/.bin/jest eslint + test262-prepare: + name: test262 - Prepare for running in parallel + runs-on: ubuntu-latest + steps: + - name: Checkout Babel + uses: actions/checkout@v3 + - name: Use Node.js latest + uses: actions/setup-node@v3 + - name: Checkout test runner + uses: actions/checkout@v3 + with: + repository: babel/babel-test262-runner + path: babel-test262-runner + - name: Download test262 + run: make bootstrap-test262 + - name: Prepare test262 chunks + run: | + cd babel-test262-runner + npm ci + node lib/prepare-chunks 4 > ~/test262-chunks.json + env: + TEST262_PATH: ../build/test262 + - name: Upload chunks artifact + uses: actions/upload-artifact@v3 + with: + name: test262-chunks + path: ~/test262-chunks.json + test262: name: test262 + strategy: + matrix: + chunk: [0, 1, 2, 3] runs-on: ubuntu-latest - needs: build + needs: [build, test262-prepare] steps: - name: Checkout Babel uses: actions/checkout@v3 @@ -514,14 +545,51 @@ jobs: node lib/download-node - name: Download test262 run: make bootstrap-test262 + - name: Download chunks file + uses: actions/download-artifact@v3 + with: + name: test262-chunks - name: Run test262 run: | cd babel-test262-runner - node lib/run-tests I_AM_SURE | tee ~/test262.tap + node lib/run-tests I_AM_SURE | tee ~/test262-${{ matrix.chunk }}.tap env: BABEL_PATH: ".." TEST262_PATH: ../build/test262 THREADS: 4 + CHUNKS_FILE: ../test262-chunks.json + CHUNK: ${{ matrix.chunk }} + - name: Create artifact with report results + uses: actions/upload-artifact@v3 + with: + name: test262-result-${{ matrix.chunk }} + path: ~/test262-${{ matrix.chunk }}.tap + + test262-finish: + name: test262 - Analyze results + runs-on: ubuntu-latest + needs: test262 + steps: + - uses: actions/download-artifact@v3 + with: + name: test262-result-0 + - uses: actions/download-artifact@v3 + with: + name: test262-result-1 + - uses: actions/download-artifact@v3 + with: + name: test262-result-2 + - uses: actions/download-artifact@v3 + with: + name: test262-result-3 + - name: Merge chunk results + run: | + cat <(test262-0.tap) + <(test262-1.tap) + <(test262-2.tap) + <(test262-3.tap) + | npx tap-merge + > ~/test262.tap - name: Create artifact with report results uses: actions/upload-artifact@v3 with: From c5564e4976b7c721dd70eb96b9ad6b6c3dc82499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Mon, 3 Oct 2022 20:12:00 +0200 Subject: [PATCH 04/10] Fix merging --- .github/workflows/ci.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c744c8d958ee..7cc32e2a4420 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -584,17 +584,12 @@ jobs: name: test262-result-3 - name: Merge chunk results run: | - cat <(test262-0.tap) - <(test262-1.tap) - <(test262-2.tap) - <(test262-3.tap) - | npx tap-merge - > ~/test262.tap + cat test262-0.tap test262-1.tap test262-2.tap test262-3.tap | npx tap-merge > test262.tap - name: Create artifact with report results uses: actions/upload-artifact@v3 with: name: test262-result - path: ~/test262.tap + path: test262.tap - name: Download previous results run: | cd babel-test262-runner From 23f12afe5b07ef80d0f1c45442ea181c6ef076d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Mon, 3 Oct 2022 22:22:32 +0200 Subject: [PATCH 05/10] Fix --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7cc32e2a4420..dca9fe1d6a8e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -590,6 +590,11 @@ jobs: with: name: test262-result path: test262.tap + - name: Checkout test runner + uses: actions/checkout@v3 + with: + repository: babel/babel-test262-runner + path: babel-test262-runner - name: Download previous results run: | cd babel-test262-runner From 1e63a93e59f4d886895bafd55244f89498d07e00 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Mon, 3 Oct 2022 23:06:13 +0200 Subject: [PATCH 06/10] More parallel --- .github/workflows/ci.yml | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dca9fe1d6a8e..5b86c328bef8 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -504,7 +504,7 @@ jobs: run: | cd babel-test262-runner npm ci - node lib/prepare-chunks 4 > ~/test262-chunks.json + node lib/prepare-chunks 8 > ~/test262-chunks.json env: TEST262_PATH: ../build/test262 - name: Upload chunks artifact @@ -512,12 +512,13 @@ jobs: with: name: test262-chunks path: ~/test262-chunks.json + retention-days: 3 test262: name: test262 strategy: matrix: - chunk: [0, 1, 2, 3] + chunk: [0, 1, 2, 3, 4, 5, 6, 7] runs-on: ubuntu-latest needs: [build, test262-prepare] steps: @@ -564,6 +565,7 @@ jobs: with: name: test262-result-${{ matrix.chunk }} path: ~/test262-${{ matrix.chunk }}.tap + retention-days: 3 test262-finish: name: test262 - Analyze results @@ -571,20 +573,26 @@ jobs: needs: test262 steps: - uses: actions/download-artifact@v3 - with: - name: test262-result-0 + with: { name: test262-result-0 } - uses: actions/download-artifact@v3 - with: - name: test262-result-1 + with: { name: test262-result-1 } - uses: actions/download-artifact@v3 - with: - name: test262-result-2 + with: { name: test262-result-2 } - uses: actions/download-artifact@v3 - with: - name: test262-result-3 + with: { name: test262-result-3 } + - uses: actions/download-artifact@v3 + with: { name: test262-result-4 } + - uses: actions/download-artifact@v3 + with: { name: test262-result-5 } + - uses: actions/download-artifact@v3 + with: { name: test262-result-6 } + - uses: actions/download-artifact@v3 + with: { name: test262-result-7 } - name: Merge chunk results run: | - cat test262-0.tap test262-1.tap test262-2.tap test262-3.tap | npx tap-merge > test262.tap + cat test262-0.tap test262-1.tap test262-2.tap test262-3.tap \ + test262-4.tap test262-5.tap test262-6.tap test262-7.tap \ + | npx tap-merge > test262.tap - name: Create artifact with report results uses: actions/upload-artifact@v3 with: From 3d7b7370c4e54889ae885c8d8fac2381870cceda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Tue, 4 Oct 2022 18:31:13 +0200 Subject: [PATCH 07/10] Fix "Download previous results", don't compare yet --- .github/workflows/ci.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5b86c328bef8..005fcbc0a959 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -606,10 +606,11 @@ jobs: - name: Download previous results run: | cd babel-test262-runner + npm ci node lib/download-main-artifact/index.mjs ~/test262-main.tap env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: Compare previous main branch & current results - run: | - cd babel-test262-runner - node lib/compare-results ~/test262-main.tap ~/test262.tap | tee ~/diff.tap + # - name: Compare previous main branch & current results + # run: | + # cd babel-test262-runner + # node lib/compare-results ~/test262-main.tap ~/test262.tap | tee ~/diff.tap From e7ec1d051b1c68423c9ef17604aa9f002c0a3bec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Tue, 4 Oct 2022 18:33:44 +0200 Subject: [PATCH 08/10] Update .github/workflows/ci.yml --- .github/workflows/ci.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 005fcbc0a959..d9d599615d78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -603,13 +603,13 @@ jobs: with: repository: babel/babel-test262-runner path: babel-test262-runner - - name: Download previous results - run: | - cd babel-test262-runner - npm ci - node lib/download-main-artifact/index.mjs ~/test262-main.tap - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + # - name: Download previous results + # run: | + # cd babel-test262-runner + # npm ci + # node lib/download-main-artifact/index.mjs ~/test262-main.tap + # env: + # GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # - name: Compare previous main branch & current results # run: | # cd babel-test262-runner From cc7a51e9862ce319d885dff636b7470c86c18e79 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Wed, 5 Oct 2022 21:44:19 +0200 Subject: [PATCH 09/10] Try running macos which has 3 cores instead of 2 --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index d9d599615d78..a5dbee9a9e97 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -519,7 +519,7 @@ jobs: strategy: matrix: chunk: [0, 1, 2, 3, 4, 5, 6, 7] - runs-on: ubuntu-latest + runs-on: macos-latest needs: [build, test262-prepare] steps: - name: Checkout Babel From a1b8bc4cd7f087eb3b88d4021231855ea33213b2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nicol=C3=B2=20Ribaudo?= Date: Wed, 5 Oct 2022 22:49:19 +0200 Subject: [PATCH 10/10] Revert "Try running macos which has 3 cores instead of 2" This reverts commit cc7a51e9862ce319d885dff636b7470c86c18e79. --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a5dbee9a9e97..d9d599615d78 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -519,7 +519,7 @@ jobs: strategy: matrix: chunk: [0, 1, 2, 3, 4, 5, 6, 7] - runs-on: macos-latest + runs-on: ubuntu-latest needs: [build, test262-prepare] steps: - name: Checkout Babel