Skip to content

Commit

Permalink
feat(vitest): add github actions reporter (#5093)
Browse files Browse the repository at this point in the history
  • Loading branch information
hi-ogawa committed Feb 7, 2024
1 parent 0f2d9ff commit 40afbe3
Show file tree
Hide file tree
Showing 9 changed files with 168 additions and 10 deletions.
10 changes: 10 additions & 0 deletions docs/guide/reporters.md
Expand Up @@ -445,6 +445,16 @@ export default defineConfig({
```
:::

### Github Actions Reporter <Badge type="info">1.3.0+</Badge>

Output [workflow commands](https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message)
to provide annotations for test failures. This reporter is automatically enabled when `process.env.GITHUB_ACTIONS === 'true'`, thus it doesn't require any configuration.

<img alt="Github Actions" img-dark src="https://github.com/vitest-dev/vitest/assets/4232207/336cddc2-df6b-4b8a-8e72-4d00010e37f5">
<img alt="Github Actions" img-light src="https://github.com/vitest-dev/vitest/assets/4232207/ce8447c1-0eab-4fe1-abef-d0d322290dca">



## Custom Reporters

You can use third-party custom reporters installed from NPM by specifying their package name in the reporters' option:
Expand Down
4 changes: 4 additions & 0 deletions packages/vitest/src/node/config.ts
Expand Up @@ -422,6 +422,10 @@ export function resolveConfig(
if (!resolved.reporters.length)
resolved.reporters.push(['default', {}])

// automatically enable github-actions reporter
if (process.env.GITHUB_ACTIONS === 'true' && !resolved.reporters.some(v => Array.isArray(v) && v[0] === 'github-actions'))
resolved.reporters.push(['github-actions', {}])

if (resolved.changed)
resolved.passWithNoTests ??= true

Expand Down
19 changes: 13 additions & 6 deletions packages/vitest/src/node/error.ts
Expand Up @@ -22,7 +22,11 @@ interface PrintErrorOptions {
showCodeFrame?: boolean
}

export async function printError(error: unknown, project: WorkspaceProject | undefined, options: PrintErrorOptions) {
interface PrintErrorResult {
nearest?: ParsedStack
}

export async function printError(error: unknown, project: WorkspaceProject | undefined, options: PrintErrorOptions): Promise<PrintErrorResult | undefined> {
const { showCodeFrame = true, fullStack = false, type } = options
const logger = options.logger
let e = error as ErrorWithDiff
Expand All @@ -43,8 +47,10 @@ export async function printError(error: unknown, project: WorkspaceProject | und
}

// Error may have occured even before the configuration was resolved
if (!project)
return printErrorMessage(e, logger)
if (!project) {
printErrorMessage(e, logger)
return
}

const parserOptions: StackTraceParserOptions = {
// only browser stack traces require remapping
Expand Down Expand Up @@ -85,7 +91,7 @@ export async function printError(error: unknown, project: WorkspaceProject | und
logger.error(c.yellow(e.frame))
}
else {
printStack(project, stacks, nearest, errorProperties, (s) => {
printStack(logger, project, stacks, nearest, errorProperties, (s) => {
if (showCodeFrame && s === nearest && nearest) {
const sourceCode = readFileSync(nearest.file, 'utf-8')
logger.error(generateCodeFrame(sourceCode.length > 100_000 ? sourceCode : logger.highlight(nearest.file, sourceCode), 4, s))
Expand Down Expand Up @@ -116,6 +122,8 @@ export async function printError(error: unknown, project: WorkspaceProject | und
}

handleImportOutsideModuleError(e.stack || e.stackStr || '', logger)

return { nearest }
}

function printErrorType(type: string, ctx: Vitest) {
Expand Down Expand Up @@ -229,14 +237,13 @@ function printErrorMessage(error: ErrorWithDiff, logger: Logger) {
}

function printStack(
logger: Logger,
project: WorkspaceProject,
stack: ParsedStack[],
highlight: ParsedStack | undefined,
errorProperties: Record<string, unknown>,
onStack?: ((stack: ParsedStack) => void),
) {
const logger = project.ctx.logger

for (const frame of stack) {
const color = frame === highlight ? c.cyan : c.gray
const path = relative(project.config.root, frame.file)
Expand Down
4 changes: 2 additions & 2 deletions packages/vitest/src/node/logger.ts
Expand Up @@ -82,10 +82,10 @@ export class Logger {
this.console.log(`${CURSOR_TO_START}${ERASE_DOWN}${log}`)
}

printError(err: unknown, options: ErrorOptions = {}) {
async printError(err: unknown, options: ErrorOptions = {}) {
const { fullStack = false, type } = options
const project = options.project ?? this.ctx.getCoreWorkspaceProject() ?? this.ctx.projects[0]
return printError(err, project, {
await printError(err, project, {
fullStack,
type,
showCodeFrame: true,
Expand Down
116 changes: 116 additions & 0 deletions packages/vitest/src/node/reporters/github-actions.ts
@@ -0,0 +1,116 @@
import { Console } from 'node:console'
import { Writable } from 'node:stream'
import { getTasks } from '@vitest/runner/utils'
import stripAnsi from 'strip-ansi'
import type { File, Reporter, Vitest } from '../../types'
import { getFullName } from '../../utils'
import { printError } from '../error'
import { Logger } from '../logger'
import type { WorkspaceProject } from '../workspace'

export class GithubActionsReporter implements Reporter {
ctx: Vitest = undefined!

onInit(ctx: Vitest) {
this.ctx = ctx
}

async onFinished(files: File[] = [], errors: unknown[] = []) {
// collect all errors and associate them with projects
const projectErrors = new Array<{ project: WorkspaceProject; title: string; error: unknown }>()
for (const error of errors) {
projectErrors.push({
project: this.ctx.getCoreWorkspaceProject(),
title: 'Unhandled error',
error,
})
}
for (const file of files) {
const tasks = getTasks(file)
const project = this.ctx.getProjectByTaskId(file.id)
for (const task of tasks) {
if (task.result?.state !== 'fail')
continue

const title = getFullName(task, ' > ')
for (const error of task.result?.errors ?? []) {
projectErrors.push({
project,
title,
error,
})
}
}
}

// format errors via `printError`
for (const { project, title, error } of projectErrors) {
const result = await printErrorWrapper(error, this.ctx, project)
const stack = result?.nearest
if (!stack)
continue
const formatted = formatMessage({
command: 'error',
properties: {
file: stack.file,
title,
line: String(stack.line),
column: String(stack.column),
},
message: stripAnsi(result.output),
})
this.ctx.logger.log(`\n${formatted}`)
}
}
}

// use Logger with custom Console to extract messgage from `processError` util
// TODO: maybe refactor `processError` to require single function `(message: string) => void` instead of full Logger?
async function printErrorWrapper(error: unknown, ctx: Vitest, project: WorkspaceProject) {
let output = ''
const writable = new Writable({
write(chunk, _encoding, callback) {
output += String(chunk)
callback()
},
})
const result = await printError(error, project, {
showCodeFrame: false,
logger: new Logger(ctx, new Console(writable, writable)),
})
return { nearest: result?.nearest, output }
}

// workflow command formatting based on
// https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#setting-an-error-message
// https://github.com/actions/toolkit/blob/f1d9b4b985e6f0f728b4b766db73498403fd5ca3/packages/core/src/command.ts#L80-L85
function formatMessage({
command,
properties,
message,
}: {
command: string
properties: Record<string, string>
message: string
}): string {
let result = `::${command}`
Object.entries(properties).forEach(([k, v], i) => {
result += i === 0 ? ' ' : ','
result += `${k}=${escapeProperty(v)}`
})
result += `::${escapeData(message)}`
return result
}

function escapeData(s: string): string {
return s.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A')
}

function escapeProperty(s: string): string {
return s
.replace(/%/g, '%25')
.replace(/\r/g, '%0D')
.replace(/\n/g, '%0A')
.replace(/:/g, '%3A')
.replace(/,/g, '%2C')
}
3 changes: 3 additions & 0 deletions packages/vitest/src/node/reporters/index.ts
Expand Up @@ -8,6 +8,7 @@ import { TapReporter } from './tap'
import { type JUnitOptions, JUnitReporter } from './junit'
import { TapFlatReporter } from './tap-flat'
import { HangingProcessReporter } from './hanging-process'
import { GithubActionsReporter } from './github-actions'
import type { BaseReporter } from './base'

export {
Expand All @@ -20,6 +21,7 @@ export {
JUnitReporter,
TapFlatReporter,
HangingProcessReporter,
GithubActionsReporter,
}
export type { BaseReporter, Reporter }

Expand All @@ -35,6 +37,7 @@ export const ReportersMap = {
'tap-flat': TapFlatReporter,
'junit': JUnitReporter,
'hanging-process': HangingProcessReporter,
'github-actions': GithubActionsReporter,
}

export type BuiltinReporters = keyof typeof ReportersMap
Expand Down
2 changes: 1 addition & 1 deletion test/reporters/package.json
Expand Up @@ -3,7 +3,7 @@
"type": "module",
"private": true,
"scripts": {
"test": "NO_COLOR=1 vitest run"
"test": "NO_COLOR=1 GITHUB_ACTIONS=false vitest run"
},
"devDependencies": {
"flatted": "^3.2.9",
Expand Down
18 changes: 18 additions & 0 deletions test/reporters/tests/github-actions.test.ts
@@ -0,0 +1,18 @@
import { resolve } from 'pathe'
import { expect, test } from 'vitest'
import { runVitest } from '../../test-utils'
import { GithubActionsReporter } from '../../../packages/vitest/src/node/reporters'

test(GithubActionsReporter, async () => {
let { stdout, stderr } = await runVitest(
{ reporters: new GithubActionsReporter(), root: './fixtures' },
['some-failing.test.ts'],
)
stdout = stdout.replace(resolve(__dirname, '..').replace(/:/g, '%3A'), '__TEST_DIR__')
expect(stdout).toMatchInlineSnapshot(`
"
::error file=__TEST_DIR__/fixtures/some-failing.test.ts,title=some-failing.test.ts > 3 + 3 = 7,line=8,column=17::AssertionError: expected 6 to be 7 // Object.is equality%0A%0A- Expected%0A+ Received%0A%0A- 7%0A+ 6%0A%0A ❯ some-failing.test.ts:8:17%0A%0A
"
`)
expect(stderr).toBe('')
})
2 changes: 1 addition & 1 deletion test/ui/package.json
Expand Up @@ -3,7 +3,7 @@
"type": "module",
"private": true,
"scripts": {
"test-e2e": "playwright test",
"test-e2e": "GITHUB_ACTIONS=false playwright test",
"test-fixtures": "vitest"
},
"devDependencies": {
Expand Down

0 comments on commit 40afbe3

Please sign in to comment.