Experimentally expose internal events for custom reporters

Add a new `observeRunsFromConfig` experiment, which allows a test run to be observed by a function installed through an `ava.config.*` file.

The function has access to AVA's internal events, which can then be used to report to a file.

AVA's internal event structure is not currently covered by any SemVer guarantees, which is why this feature requires the experimental opt-in.

Does not currently support watch mode. Only the first run is observed.

entrypoints/internal.d.mts

@@ -0,0 +1,7 @@
+import type {StateChangeEvent} from '../types/state-change-events.d';
+
+export type Event = StateChangeEvent;
+
+export type ObservedRun = {
+	events: AsyncIterableIterator<Event>;
+};

lib/api-event-iterator.js

@@ -0,0 +1,12 @@
+export async function * asyncEventIteratorFromApi(api) {
+	// TODO: support multiple runs (watch mode)
+	const {value: plan} = await api.events('run').next();
+
+	for await (const stateChange of plan.status.events('stateChange')) {
+		yield stateChange;
+
+		if (stateChange.type === 'end' || stateChange.type === 'interrupt') {
+			break;
+		}
+	}
+}

lib/cli.js

@@ -8,6 +8,7 @@ import figures from 'figures';
 import yargs from 'yargs';
 import {hideBin} from 'yargs/helpers'; // eslint-disable-line n/file-extension-in-import
 
+import {asyncEventIteratorFromApi} from './api-event-iterator.js';
 import Api from './api.js';
 import {chalk} from './chalk.js';
 import validateEnvironmentVariables from './environment-variables.js';
@@ -470,6 +471,12 @@ export default async function loadCli() { // eslint-disable-line complexity
 		});
 	}
 
+	if (combined.observeRun && experiments.observeRunsFromConfig) {
+		combined.observeRun({
+			events: asyncEventIteratorFromApi(api),
+		});
+	}
+
 	api.on('run', plan => {
 		reporter.startRun(plan);
 

lib/load-config.js

@@ -8,7 +8,7 @@ import {packageConfig, packageJsonPath} from 'package-config';
 
 const NO_SUCH_FILE = Symbol('no ava.config.js file');
 const MISSING_DEFAULT_EXPORT = Symbol('missing default export');
-const EXPERIMENTS = new Set();
+const EXPERIMENTS = new Set(['observeRunsFromConfig']);
 
 const importConfig = async ({configFile, fileForErrorMessage}) => {
 	const {default: config = MISSING_DEFAULT_EXPORT} = await import(url.pathToFileURL(configFile));

package.json

@@ -29,6 +29,9 @@
 				"types": "./entrypoints/plugin.d.cts",
 				"default": "./entrypoints/plugin.cjs"
 			}
+		},
+		"./internal": {
+			"types": "./entrypoints/internal.d.mts"
 		}
 	},
 	"type": "module",

test/internal-events/fixtures/.gitignore

@@ -0,0 +1 @@
+internal-events.json

test/internal-events/fixtures/ava.config.js

@@ -0,0 +1,19 @@
+import fs from 'node:fs/promises';
+
+const internalEvents = [];
+
+export default {
+	files: [
+		'test.js',
+	],
+	nonSemVerExperiments: {
+		observeRunsFromConfig: true,
+	},
+	async observeRun(run) {
+		for await (const event of run.events) {
+			internalEvents.push(event);
+		}
+
+		await fs.writeFile('internal-events.json', JSON.stringify(internalEvents));
+	},
+};

test/internal-events/fixtures/package.json

@@ -0,0 +1,3 @@
+{
+	"type": "module"
+}

test/internal-events/fixtures/test.js

@@ -0,0 +1,5 @@
+import test from 'ava';
+
+test('placeholder', t => {
+	t.pass();
+});

test/internal-events/test.js

@@ -0,0 +1,28 @@
+import fs from 'node:fs/promises';
+import {fileURLToPath} from 'node:url';
+
+import test from '@ava/test';
+
+import {fixture} from '../helpers/exec.js';
+
+test('internal events are emitted', async t => {
+	await fixture();
+
+	const result = JSON.parse(await fs.readFile(fileURLToPath(new URL('fixtures/internal-events.json', import.meta.url))));
+
+	t.like(result[0], {
+		type: 'starting',
+		testFile: fileURLToPath(new URL('fixtures/test.js', import.meta.url)),
+	});
+
+	const testPassedEvent = result.find(event => event.type === 'test-passed');
+	t.like(testPassedEvent, {
+		type: 'test-passed',
+		title: 'placeholder',
+		testFile: fileURLToPath(new URL('fixtures/test.js', import.meta.url)),
+	});
+
+	t.like(result.at(-1), {
+		type: 'end',
+	});
+});

types/state-change-events.d.cts

@@ -0,0 +1,143 @@
+type ErrorSource = {
+	isDependency: boolean;
+	isWithinProject: boolean;
+	file: string;
+	line: number;
+};
+
+type SerializedErrorBase = {
+	message: string;
+	name: string;
+	originalError: unknown;
+	stack: string;
+};
+
+type AggregateSerializedError = SerializedErrorBase & {
+	type: 'aggregate';
+	errors: SerializedError[];
+};
+
+type NativeSerializedError = SerializedErrorBase & {
+	type: 'native';
+	source: ErrorSource | undefined;
+};
+
+type AvaSerializedError = SerializedErrorBase & {
+	type: 'ava';
+	assertion: string;
+	improperUsage: unknown | undefined;
+	formattedCause: unknown | undefined;
+	formattedDetails: unknown | unknown[];
+	source: ErrorSource | undefined;
+};
+
+type SerializedError = AggregateSerializedError | NativeSerializedError | AvaSerializedError;
+
+export type StateChangeEvent = {
+	type: 'starting';
+	testFile: string;
+} | {
+	type: 'stats';
+	stats: {
+		byFile: Map<string, {
+			declaredTests: number;
+			failedHooks: number;
+			failedTests: number;
+			internalErrors: number;
+			remainingTests: number;
+			passedKnownFailingTests: number;
+			passedTests: number;
+			selectedTests: number;
+			selectingLines: boolean;
+			skippedTests: number;
+			todoTests: number;
+			uncaughtExceptions: number;
+			unhandledRejections: number;
+		}>;
+		declaredTests: number;
+		failedHooks: number;
+		failedTests: number;
+		failedWorkers: number;
+		files: number;
+		parallelRuns: {
+			currentIndex: number;
+			totalRuns: number;
+		} | undefined;
+		finishedWorkers: number;
+		internalErrors: number;
+		remainingTests: number;
+		passedKnownFailingTests: number;
+		passedTests: number;
+		selectedTests: number;
+		sharedWorkerErrors: number;
+		skippedTests: number;
+		timedOutTests: number;
+		timeouts: number;
+		todoTests: number;
+		uncaughtExceptions: number;
+		unhandledRejections: number;
+	};
+} | {
+	type: 'declared-test';
+	title: string;
+	knownFailing: boolean;
+	todo: boolean;
+	testFile: string;
+} | {
+	type: 'selected-test';
+	title: string;
+	knownFailing: boolean;
+	skip: boolean;
+	todo: boolean;
+	testFile: string;
+} | {
+	type: 'test-register-log-reference';
+	title: string;
+	logs: string[];
+	testFile: string;
+} | {
+	type: 'test-passed';
+	title: string;
+	duration: number;
+	knownFailing: boolean;
+	logs: string[];
+	testFile: string;
+} | {
+	type: 'test-failed';
+	title: string;
+	err: SerializedError;
+	duration: number;
+	knownFailing: boolean;
+	logs: string[];
+	testFile: string;
+} | {
+	type: 'worker-finished';
+	forcedExit: boolean;
+	testFile: string;
+} | {
+	type: 'worker-failed';
+	nonZeroExitCode?: boolean;
+	signal?: string;
+	err?: SerializedError;
+} | {
+	type: 'touched-files';
+	files: {
+		changedFiles: string[];
+		temporaryFiles: string[];
+	};
+} | {
+	type: 'worker-stdout';
+	chunk: Uint8Array;
+	testFile: string;
+} | {
+	type: 'worker-stderr';
+	chunk: Uint8Array;
+	testFile: string;
+} | {
+	type: 'timeout';
+	period: number;
+	pendingTests: Map<string, Set<string>>;
+}
+| {
+	type: 'end';
+};