From df981ea3789d5a5c6db18c055e13833db64bd4f5 Mon Sep 17 00:00:00 2001 From: Benjamin Coe Date: Sat, 17 Oct 2020 09:13:33 -0700 Subject: [PATCH 01/12] test: add windows and C++ coverage --- .github/workflows/coverage-linux.yml | 1 - .github/workflows/coverage-windows.yml | 42 ++++++++++++++++++++++++++ 2 files changed, 42 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/coverage-windows.yml diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 5ebd89115b4230..b804a4fdae8338 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -1,4 +1,3 @@ -# TODO(bcoe): add similar job for Windows coverage. name: coverage-linux on: diff --git a/.github/workflows/coverage-windows.yml b/.github/workflows/coverage-windows.yml new file mode 100644 index 00000000000000..ee31b44c41fd6b --- /dev/null +++ b/.github/workflows/coverage-windows.yml @@ -0,0 +1,42 @@ +name: coverage-windows + +on: + pull_request: + push: + branches: + - master + +env: + PYTHON_VERSION: 3.9 + FLAKY_TESTS: dontcare + +jobs: + coverage-windows: + runs-on: windows-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v2 + with: + python-version: ${{ env.PYTHON_VERSION }} + - name: Install deps + run: choco install nasm + - name: Environment Information + run: npx envinfo + - name: Build + run: ./vcbuild.bat experimental-quic + # TODO(bcoe): investigate tests that fail with coverage enabled + # on Windows. + - name: Test + run: ./vcbuild.bat test-ci-js; node -e 'process.exit(0)' + env: + NODE_V8_COVERAGE: ./coverage/tmp + - name: Report + run: npx c8 report + - name: Clean tmp + run: npx rimraf ./coverage/tmp + - name: Upload + uses: codecov/codecov-action@v1 + with: + directory: ./coverage + From d91b81d87a6fa260451cda124d6c5d72b2e19acd Mon Sep 17 00:00:00 2001 From: bcoe Date: Sat, 17 Oct 2020 21:09:35 -0700 Subject: [PATCH 02/12] build: improve coverage configuration --- .github/workflows/coverage-windows.yml | 1 - codecov.yml | 8 ++++++++ 2 files changed, 8 insertions(+), 1 deletion(-) create mode 100644 codecov.yml diff --git a/.github/workflows/coverage-windows.yml b/.github/workflows/coverage-windows.yml index ee31b44c41fd6b..0131b7715a8f8e 100644 --- a/.github/workflows/coverage-windows.yml +++ b/.github/workflows/coverage-windows.yml @@ -39,4 +39,3 @@ jobs: uses: codecov/codecov-action@v1 with: directory: ./coverage - diff --git a/codecov.yml b/codecov.yml new file mode 100644 index 00000000000000..7474e10ba06523 --- /dev/null +++ b/codecov.yml @@ -0,0 +1,8 @@ +comment: + # Only show diff and files changed: + layout: "diff, files" + # Don't post if no changes in coverage: + require_changes: true +notify: + # Wait for all coverage builds: + after_n_builds: 2 From 656160fd853d07c743d80599418cdc06fb204ae6 Mon Sep 17 00:00:00 2001 From: bcoe Date: Sat, 17 Oct 2020 22:04:24 -0700 Subject: [PATCH 03/12] chore: attempt to build with gcov --- .github/workflows/coverage-linux.yml | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index b804a4fdae8338..3f15fd68f0f993 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -11,7 +11,6 @@ env: FLAKY_TESTS: dontcare jobs: - # TODO(bcoe): add support for C++ coverage. coverage-linux: runs-on: ubuntu-latest steps: @@ -23,13 +22,18 @@ jobs: - name: Environment Information run: npx envinfo - name: Build - run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn" + run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --coverage" # TODO(bcoe): fix the couple tests that fail with the inspector enabled. # The cause is most likely coverage's use of the inspector. - name: Test run: NODE_V8_COVERAGE=coverage/tmp make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0 - - name: Report + - name: Report js run: npx c8 report --check-coverage + - name: Report gcov + run: cd out && ../gcovr/scripts/gcovr \ + --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v \ + -r Release/obj.target --xml -o ../coverage/coverage.xml \ + --gcov-executable=gcov - name: Output file count run: ls -l coverage/tmp/ | wc -l - name: Clean tmp From 9eac50be90c16e4a41df44bde9cc8494b682625f Mon Sep 17 00:00:00 2001 From: bcoe Date: Sat, 17 Oct 2020 22:16:24 -0700 Subject: [PATCH 04/12] build: ensure gcovr tool exists --- .github/workflows/coverage-linux.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 3f15fd68f0f993..f37d6fe376170d 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -21,6 +21,12 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Environment Information run: npx envinfo + - name: Clone gcovr + run: git clone -b 3.4 --depth=1 --single-branch https://github.com/gcovr/gcovr.git + - name: Clone patch for gcovr + run: git clone --depth=1 --single-branch https://github.com/nodejs/build.git + - name: Patch gcovr + run: cd gcovr && patch -N -p1 < ../build/jenkins/scripts/coverage/gcovr-patches-3.4.diff - name: Build run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --coverage" # TODO(bcoe): fix the couple tests that fail with the inspector enabled. From f8692553858403d970d2a0227eab868d96c58739 Mon Sep 17 00:00:00 2001 From: bcoe Date: Sat, 17 Oct 2020 22:21:20 -0700 Subject: [PATCH 05/12] chore: tweak codecov.yml file --- codecov.yml | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/codecov.yml b/codecov.yml index 7474e10ba06523..9808848a48ead8 100644 --- a/codecov.yml +++ b/codecov.yml @@ -3,6 +3,8 @@ comment: layout: "diff, files" # Don't post if no changes in coverage: require_changes: true -notify: - # Wait for all coverage builds: - after_n_builds: 2 + +codecov: + notify: + # Wait for all coverage builds: + after_n_builds: 2 From f5d62726dfb31584cf66c6856acbbbb0c34a34ea Mon Sep 17 00:00:00 2001 From: bcoe Date: Sun, 18 Oct 2020 07:04:01 -0700 Subject: [PATCH 06/12] multi-line command had issues --- .github/workflows/coverage-linux.yml | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index f37d6fe376170d..91786b57fea117 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -21,7 +21,7 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Environment Information run: npx envinfo - - name: Clone gcovr + - name: Clone gcovr reporter run: git clone -b 3.4 --depth=1 --single-branch https://github.com/gcovr/gcovr.git - name: Clone patch for gcovr run: git clone --depth=1 --single-branch https://github.com/nodejs/build.git @@ -35,11 +35,8 @@ jobs: run: NODE_V8_COVERAGE=coverage/tmp make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0 - name: Report js run: npx c8 report --check-coverage - - name: Report gcov - run: cd out && ../gcovr/scripts/gcovr \ - --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v \ - -r Release/obj.target --xml -o ../coverage/coverage.xml \ - --gcov-executable=gcov + - name: Report C++ + run: cd out && ../gcovr/scripts/gcovr --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --gcov-executable=gcov - name: Output file count run: ls -l coverage/tmp/ | wc -l - name: Clean tmp From 40cb785bacbb862066dd2ae92a596fb628a67b39 Mon Sep 17 00:00:00 2001 From: bcoe Date: Sun, 18 Oct 2020 08:35:16 -0700 Subject: [PATCH 07/12] chore: delete more output files --- .github/workflows/coverage-linux.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 91786b57fea117..0ba3566e0f125c 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -39,8 +39,9 @@ jobs: run: cd out && ../gcovr/scripts/gcovr --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --gcov-executable=gcov - name: Output file count run: ls -l coverage/tmp/ | wc -l + # Clean temporary output from gcov and c8, so that it's not uploaded: - name: Clean tmp - run: rm -rf coverage/tmp + run: rm -rf coverage/tmp && rm -rf out - name: Upload uses: codecov/codecov-action@v1 with: From fc7c05c356531de604155bf7030721df058c0940 Mon Sep 17 00:00:00 2001 From: bcoe Date: Sun, 18 Oct 2020 10:15:02 -0700 Subject: [PATCH 08/12] build: run test-cov rather than test-ci --- .github/workflows/coverage-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 0ba3566e0f125c..f647253cfc87c4 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -32,7 +32,7 @@ jobs: # TODO(bcoe): fix the couple tests that fail with the inspector enabled. # The cause is most likely coverage's use of the inspector. - name: Test - run: NODE_V8_COVERAGE=coverage/tmp make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0 + run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0 - name: Report js run: npx c8 report --check-coverage - name: Report C++ From 5939514726ebb234c75b2fd6e2dfb3b1a4fa0d1a Mon Sep 17 00:00:00 2001 From: bcoe Date: Mon, 19 Oct 2020 19:57:25 -0700 Subject: [PATCH 09/12] chore: address review --- .github/workflows/coverage-linux.yml | 20 +++++++++++++------- .github/workflows/coverage-windows.yml | 12 +++++++++++- 2 files changed, 24 insertions(+), 8 deletions(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index f647253cfc87c4..c6b0bd3cfd6f27 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -2,9 +2,19 @@ name: coverage-linux on: pull_request: + paths-ignore: + - 'doc/**' + - 'deps/**' + - 'benchmark/**' + - 'tools/**' push: branches: - master + paths-ignore: + - 'doc/**' + - 'deps/**' + - 'benchmark/**' + - 'tools/**' env: PYTHON_VERSION: 3.9 @@ -21,12 +31,8 @@ jobs: python-version: ${{ env.PYTHON_VERSION }} - name: Environment Information run: npx envinfo - - name: Clone gcovr reporter - run: git clone -b 3.4 --depth=1 --single-branch https://github.com/gcovr/gcovr.git - - name: Clone patch for gcovr - run: git clone --depth=1 --single-branch https://github.com/nodejs/build.git - - name: Patch gcovr - run: cd gcovr && patch -N -p1 < ../build/jenkins/scripts/coverage/gcovr-patches-3.4.diff + - name: Install gcovr + run: pip install gcovr - name: Build run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --coverage" # TODO(bcoe): fix the couple tests that fail with the inspector enabled. @@ -36,7 +42,7 @@ jobs: - name: Report js run: npx c8 report --check-coverage - name: Report C++ - run: cd out && ../gcovr/scripts/gcovr --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --gcov-executable=gcov + run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --filter='' - name: Output file count run: ls -l coverage/tmp/ | wc -l # Clean temporary output from gcov and c8, so that it's not uploaded: diff --git a/.github/workflows/coverage-windows.yml b/.github/workflows/coverage-windows.yml index 0131b7715a8f8e..ac0ef7b6e6c018 100644 --- a/.github/workflows/coverage-windows.yml +++ b/.github/workflows/coverage-windows.yml @@ -2,9 +2,19 @@ name: coverage-windows on: pull_request: + paths-ignore: + - 'doc/**' + - 'deps/**' + - 'benchmark/**' + - 'tools/**' push: branches: - master + paths-ignore: + - 'doc/**' + - 'deps/**' + - 'benchmark/**' + - 'tools/**' env: PYTHON_VERSION: 3.9 @@ -24,7 +34,7 @@ jobs: - name: Environment Information run: npx envinfo - name: Build - run: ./vcbuild.bat experimental-quic + run: ./vcbuild.bat # TODO(bcoe): investigate tests that fail with coverage enabled # on Windows. - name: Test From 71f998af1f4f5d8998aacdb53598bc69754a0a76 Mon Sep 17 00:00:00 2001 From: bcoe Date: Mon, 19 Oct 2020 20:39:31 -0700 Subject: [PATCH 10/12] chore: fiddle with gcovr settings --- .github/workflows/coverage-linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index c6b0bd3cfd6f27..4349d38aa9902f 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -32,7 +32,7 @@ jobs: - name: Environment Information run: npx envinfo - name: Install gcovr - run: pip install gcovr + run: pip install gcovr==4.2 - name: Build run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --coverage" # TODO(bcoe): fix the couple tests that fail with the inspector enabled. @@ -42,7 +42,7 @@ jobs: - name: Report js run: npx c8 report --check-coverage - name: Report C++ - run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --filter='' + run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|obj|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --filter='' --root=$(node -e "console.info(path.resolve(process.cwd(), '../'))") - name: Output file count run: ls -l coverage/tmp/ | wc -l # Clean temporary output from gcov and c8, so that it's not uploaded: From 15d170e8a2bc7a9d658076c984757860eaab1071 Mon Sep 17 00:00:00 2001 From: bcoe Date: Mon, 19 Oct 2020 20:53:46 -0700 Subject: [PATCH 11/12] chore: simplify command --- .github/workflows/coverage-linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 4349d38aa9902f..422dd477ec64d7 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -42,7 +42,7 @@ jobs: - name: Report js run: npx c8 report --check-coverage - name: Report C++ - run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|obj|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --filter='' --root=$(node -e "console.info(path.resolve(process.cwd(), '../'))") + run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|obj|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --root=$(cd ../; pwd) - name: Output file count run: ls -l coverage/tmp/ | wc -l # Clean temporary output from gcov and c8, so that it's not uploaded: From c646f6d3a7c5b5a1f5b50e1b38130ec6ac28290a Mon Sep 17 00:00:00 2001 From: bcoe Date: Tue, 20 Oct 2020 08:42:29 -0700 Subject: [PATCH 12/12] build: a couple more small tweaks to reports --- .github/workflows/coverage-linux.yml | 6 ++---- .nycrc | 2 +- Makefile | 1 + 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/coverage-linux.yml b/.github/workflows/coverage-linux.yml index 422dd477ec64d7..002c2189b82459 100644 --- a/.github/workflows/coverage-linux.yml +++ b/.github/workflows/coverage-linux.yml @@ -39,12 +39,10 @@ jobs: # The cause is most likely coverage's use of the inspector. - name: Test run: NODE_V8_COVERAGE=coverage/tmp make test-cov -j2 V=1 TEST_CI_ARGS="-p dots" || exit 0 - - name: Report js + - name: Report JS run: npx c8 report --check-coverage - name: Report C++ - run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|obj|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage.xml --root=$(cd ../; pwd) - - name: Output file count - run: ls -l coverage/tmp/ | wc -l + run: cd out && gcovr --gcov-exclude='.*\b(deps|usr|out|obj|cctest|embedding)\b' -v -r Release/obj.target --xml -o ../coverage/coverage-cxx.xml --root=$(cd ../ && pwd) # Clean temporary output from gcov and c8, so that it's not uploaded: - name: Clean tmp run: rm -rf coverage/tmp && rm -rf out diff --git a/.nycrc b/.nycrc index 6192a73a77b0d2..42aad37414a2ee 100644 --- a/.nycrc +++ b/.nycrc @@ -9,7 +9,7 @@ "reporter": [ "html", "text", - "lcov" + "cobertura" ], "lines": 95, "branches": "93", diff --git a/Makefile b/Makefile index 1c69810f9a04cf..e23a479734861f 100644 --- a/Makefile +++ b/Makefile @@ -297,6 +297,7 @@ v8: .PHONY: jstest jstest: build-addons build-js-native-api-tests build-node-api-tests ## Runs addon tests and JS tests $(PYTHON) tools/test.py $(PARALLEL_ARGS) --mode=$(BUILDTYPE_LOWER) \ + $(TEST_CI_ARGS) \ --skip-tests=$(CI_SKIP_TESTS) \ $(JS_SUITES) \ $(NATIVE_SUITES)