diff --git a/packages/vitest/src/node/core.ts b/packages/vitest/src/node/core.ts
index 0ec32306e3e..bdbe27f6bf0 100644
--- a/packages/vitest/src/node/core.ts
+++ b/packages/vitest/src/node/core.ts
@@ -285,11 +285,14 @@ export class Vitest {
       return
     }
 
-    await this.report('onInit', this)
-
-    await this.initCoverageProvider()
-    await this.coverageProvider?.clean(this.config.coverage.clean)
-    await this.initBrowserProviders()
+    try {
+      await this.initCoverageProvider()
+      await this.coverageProvider?.clean(this.config.coverage.clean)
+      await this.initBrowserProviders()
+    }
+    finally {
+      await this.report('onInit', this)
+    }
 
     const files = await this.filterTestsBySource(
       await this.globTestFiles(filters),
diff --git a/packages/vitest/src/node/reporters/junit.ts b/packages/vitest/src/node/reporters/junit.ts
index a9b91817562..9437bb83331 100644
--- a/packages/vitest/src/node/reporters/junit.ts
+++ b/packages/vitest/src/node/reporters/junit.ts
@@ -75,6 +75,7 @@ export class JUnitReporter implements Reporter {
   private baseLog!: (text: string) => Promise<void>
   private logger!: IndentedLogger<Promise<void>>
   private _timeStart = new Date()
+  private fileFd?: fs.FileHandle
 
   async onInit(ctx: Vitest): Promise<void> {
     this.ctx = ctx
@@ -89,6 +90,7 @@ export class JUnitReporter implements Reporter {
         await fs.mkdir(outputDirectory, { recursive: true })
 
       const fileFd = await fs.open(this.reportFile, 'w+')
+      this.fileFd = fileFd
 
       this.baseLog = async (text: string) => await fs.writeFile(fileFd, `${text}\n`)
     }
@@ -244,5 +246,7 @@ export class JUnitReporter implements Reporter {
 
     if (this.reportFile)
       this.ctx.logger.log(`JUNIT report written to ${this.reportFile}`)
+
+    await this.fileFd?.close()
   }
 }
diff --git a/test/config/package.json b/test/config/package.json
index 2bde0e50bbe..56c95551587 100644
--- a/test/config/package.json
+++ b/test/config/package.json
@@ -2,7 +2,7 @@
   "name": "@vitest/test-config",
   "private": true,
   "scripts": {
-    "test": "vitest run test/**"
+    "test": "vitest run"
   },
   "devDependencies": {
     "execa": "^7.0.0",
diff --git a/test/config/test/failures.test.ts b/test/config/test/failures.test.ts
index 5abde7b08cb..0dcfcd0f903 100644
--- a/test/config/test/failures.test.ts
+++ b/test/config/test/failures.test.ts
@@ -1,4 +1,5 @@
 import { expect, test } from 'vitest'
+import { version } from 'vitest/package.json'
 
 import { runVitest } from './utils'
 
@@ -57,3 +58,16 @@ test('boolean browser flag without dot notation, with more dot notation options'
   expect(error).toMatch('Error: A boolean argument "--browser" was used with dot notation arguments "--browser.name".')
   expect(error).toMatch('Please specify the "--browser" argument with dot notation as well: "--browser.enabled"')
 })
+
+test('version number is printed when coverage provider fails to load', async () => {
+  const { error, output } = await runVitest('run', [
+    '--coverage.enabled',
+    '--coverage.provider',
+    'custom',
+    '--coverage.customProviderModule',
+    './non-existing-module.ts',
+  ])
+
+  expect(output).toMatch(`RUN  v${version}`)
+  expect(error).toMatch('Error: Failed to load custom CoverageProviderModule from ./non-existing-module.ts')
+})
diff --git a/test/config/test/utils.ts b/test/config/test/utils.ts
index 51e8b492249..1dc0ccabd2e 100644
--- a/test/config/test/utils.ts
+++ b/test/config/test/utils.ts
@@ -4,15 +4,17 @@ import stripAnsi from 'strip-ansi'
 export async function runVitest(mode: 'run' | 'watch', cliArguments: string[]) {
   const subprocess = execa('vitest', [mode, 'fixtures/test/', ...cliArguments])
   let error = ''
+  let output = ''
+
+  subprocess.stdout?.on('data', (data) => {
+    output += stripAnsi(data.toString())
+  })
 
   subprocess.stderr?.on('data', (data) => {
     error += stripAnsi(data.toString())
-
-    // Sometimes on Windows CI execa doesn't exit properly. Force exit when stderr is caught.
-    subprocess.kill()
   })
 
   await new Promise(resolve => subprocess.on('exit', resolve))
 
-  return { error }
+  return { output, error }
 }
diff --git a/test/config/vitest.config.ts b/test/config/vitest.config.ts
index a3f5c3ce3f0..8d362f825f9 100644
--- a/test/config/vitest.config.ts
+++ b/test/config/vitest.config.ts
@@ -2,6 +2,7 @@ import { defineConfig } from 'vitest/config'
 
 export default defineConfig({
   test: {
+    include: ['test/**.test.ts'],
     testTimeout: 60_000,
     chaiConfig: {
       truncateThreshold: 999,
diff --git a/test/coverage-test/coverage-report-tests/generic.report.test.ts b/test/coverage-test/coverage-report-tests/generic.report.test.ts
index 705a56b7b6c..beb9f8ab820 100644
--- a/test/coverage-test/coverage-report-tests/generic.report.test.ts
+++ b/test/coverage-test/coverage-report-tests/generic.report.test.ts
@@ -94,3 +94,10 @@ test('function count is correct', async () => {
   expect(functions.total).toBe(5)
   expect(functions.covered).toBe(3)
 })
+
+test('coverage provider does not conflict with built-in reporter\'s outputFile', async () => {
+  const coveragePath = resolve('./coverage')
+  const files = fs.readdirSync(coveragePath)
+
+  expect(files).toContain('junit.xml')
+})
diff --git a/test/coverage-test/testing.mjs b/test/coverage-test/testing.mjs
index 7b924d0c4a4..dab0578302a 100644
--- a/test/coverage-test/testing.mjs
+++ b/test/coverage-test/testing.mjs
@@ -17,6 +17,10 @@ const configs = [
     ].filter(Boolean),
     coverage: { enabled: true },
     browser: { enabled: isBrowser, name: 'chrome', headless: true },
+
+    // Regression vitest#3330
+    reporters: ['default', 'junit'],
+    outputFile: { junit: 'coverage/junit.xml' },
   }],
 
   // Run tests for checking coverage report contents.