From 46bef7b7716b377055260490bb6f713ffb593672 Mon Sep 17 00:00:00 2001 From: Mary Marchini Date: Fri, 31 Jul 2020 21:40:47 -0700 Subject: [PATCH] test: add Actions annotation output It's possible to annotate failures in Actions by printing "::error file={},line={},col={}::{message}". This methos is preferrable over using a problem matcher because problem matchers only allow single-line messages, whereas ::error allows multi-line messages. PR-URL: https://github.com/nodejs/node/pull/34590 Reviewed-By: Rich Trott Reviewed-By: Richard Lau --- .github/workflows/test-asan.yml | 2 +- .github/workflows/test-linux.yml | 4 +-- .github/workflows/test-macos.yml | 2 +- tools/test.py | 49 ++++++++++++++++++++++++-------- 4 files changed, 41 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test-asan.yml b/.github/workflows/test-asan.yml index 96975024e399f5..224d18463ac172 100644 --- a/.github/workflows/test-asan.yml +++ b/.github/workflows/test-asan.yml @@ -36,4 +36,4 @@ jobs: - name: Build run: make build-ci -j2 V=1 - name: Test - run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" + run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions" diff --git a/.github/workflows/test-linux.yml b/.github/workflows/test-linux.yml index fcf11259e8e11b..a5eb2351c3e5d2 100644 --- a/.github/workflows/test-linux.yml +++ b/.github/workflows/test-linux.yml @@ -27,7 +27,7 @@ jobs: - name: Build run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn" - name: Test - run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" + run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions" test-linux-with-quic: runs-on: ubuntu-latest @@ -42,4 +42,4 @@ jobs: - name: Build run: make build-ci -j2 V=1 CONFIG_FLAGS="--error-on-warn --experimental-quic" - name: Test - run: make run-ci -j2 V=1 TEST_CI_ARGS="-p dots" + run: make run-ci -j2 V=1 TEST_CI_ARGS="-p actions" diff --git a/.github/workflows/test-macos.yml b/.github/workflows/test-macos.yml index a4ef46e7ff4322..ba3bb5426da929 100644 --- a/.github/workflows/test-macos.yml +++ b/.github/workflows/test-macos.yml @@ -27,4 +27,4 @@ jobs: - name: Build run: make build-ci -j8 V=1 CONFIG_FLAGS="--error-on-warn --experimental-quic" - name: Test - run: make run-ci -j8 V=1 TEST_CI_ARGS="-p dots" + run: make run-ci -j8 V=1 TEST_CI_ARGS="-p actions" diff --git a/tools/test.py b/tools/test.py index dde7c7b2f8e466..2884bf44bd1dbe 100755 --- a/tools/test.py +++ b/tools/test.py @@ -116,6 +116,25 @@ def __init__(self, cases, flaky_tests_mode): self.lock = threading.Lock() self.shutdown_event = threading.Event() + def GetFailureOutput(self, failure): + output = [] + if failure.output.stderr: + output += ["--- stderr ---" ] + output += [failure.output.stderr.strip()] + if failure.output.stdout: + output += ["--- stdout ---"] + output += [failure.output.stdout.strip()] + output += ["Command: %s" % EscapeCommand(failure.command)] + if failure.HasCrashed(): + output += ["--- %s ---" % PrintCrashed(failure.output.exit_code)] + if failure.HasTimedOut(): + output += ["--- TIMEOUT ---"] + output = "\n".join(output) + return output + + def PrintFailureOutput(self, failure): + print(self.GetFailureOutput(failure)) + def PrintFailureHeader(self, test): if test.IsNegative(): negative_marker = '[negative] ' @@ -224,17 +243,7 @@ def Done(self): print() for failed in self.failed: self.PrintFailureHeader(failed.test) - if failed.output.stderr: - print("--- stderr ---") - print(failed.output.stderr.strip()) - if failed.output.stdout: - print("--- stdout ---") - print(failed.output.stdout.strip()) - print("Command: %s" % EscapeCommand(failed.command)) - if failed.HasCrashed(): - print("--- %s ---" % PrintCrashed(failed.output.exit_code)) - if failed.HasTimedOut(): - print("--- TIMEOUT ---") + self.PrintFailureOutput(failed) if len(self.failed) == 0: print("===") print("=== All tests succeeded") @@ -288,6 +297,21 @@ def HasRun(self, output): sys.stdout.write('.') sys.stdout.flush() +class ActionsAnnotationProgressIndicator(DotsProgressIndicator): + def GetAnnotationInfo(self, test, output): + traceback = output.stdout + output.stderr + find_full_path = re.search(r' +at .*\(.*%s:([0-9]+):([0-9]+)' % test.file, traceback) + col = line = 0 + if find_full_path: + line, col = map(int, find_full_path.groups()) + root_path = abspath(join(dirname(__file__), '../')) + os.sep + filename = test.file.replace(root_path, "") + return filename, line, col + + def PrintFailureOutput(self, failure): + output = self.GetFailureOutput(failure) + filename, line, column = self.GetAnnotationInfo(failure.test, failure.output) + print("::error file=%s,line=%d,col=%d::%s" % (filename, line, column, output.replace('\n', '%0A'))) class TapProgressIndicator(SimpleProgressIndicator): @@ -496,6 +520,7 @@ def ClearLine(self, last_line_length): PROGRESS_INDICATORS = { 'verbose': VerboseProgressIndicator, 'dots': DotsProgressIndicator, + 'actions': ActionsAnnotationProgressIndicator, 'color': ColorProgressIndicator, 'tap': TapProgressIndicator, 'mono': MonochromeProgressIndicator, @@ -1299,7 +1324,7 @@ def BuildOptions(): result.add_option('--logfile', dest='logfile', help='write test output to file. NOTE: this only applies the tap progress indicator') result.add_option("-p", "--progress", - help="The style of progress indicator (verbose, dots, color, mono, tap)", + help="The style of progress indicator (%s)" % ", ".join(PROGRESS_INDICATORS.keys()), choices=list(PROGRESS_INDICATORS.keys()), default="mono") result.add_option("--report", help="Print a summary of the tests to be run", default=False, action="store_true")