refactor: make ids in typecheck suites more reliable, store errors in…

… array (#2597)

* refactor: make ids in typecheck suites more reliable, store errors in array

* chore: cleanup

* Update packages/vitest/src/typecheck/typechecker.ts

packages/vitest/src/node/core.ts

@@ -71,7 +71,7 @@ export class Vitest {
     this.cache = new VitestCache()
     this.snapshot = new SnapshotManager({ ...resolved.snapshotOptions })
 
-    if (this.config.watch)
+    if (this.config.watch && this.mode !== 'typecheck')
       this.registerWatcher()
 
     this.vitenode = new ViteNodeServer(server, this.config)
@@ -154,8 +154,9 @@ export class Vitest {
     ) as ResolvedConfig
   }
 
-  async typecheck(filters?: string[]) {
-    const testsFilesList = await this.globTestFiles(filters)
+  async typecheck(filters: string[] = []) {
+    const { include, exclude } = this.config.typecheck
+    const testsFilesList = await this.globFiles(filters, include, exclude)
     const checker = new Typechecker(this, testsFilesList)
     this.typechecker = checker
     checker.onParseEnd(async ({ files, sourceErrors }) => {
@@ -198,6 +199,7 @@ export class Vitest {
       await this.report('onTaskUpdate', checker.getTestPacks())
       await this.report('onCollected')
     })
+    await checker.prepare()
     await checker.collectTests()
     await checker.start()
   }
@@ -562,9 +564,7 @@ export class Vitest {
     )))
   }
 
-  async globTestFiles(filters: string[] = []) {
-    const { include, exclude, includeSource } = this.config
-
+  async globFiles(filters: string[], include: string[], exclude: string[]) {
     const globOptions: fg.Options = {
       absolute: true,
       dot: true,
@@ -580,10 +580,16 @@ export class Vitest {
     if (filters.length)
       testFiles = testFiles.filter(i => filters.some(f => i.includes(f)))
 
+    return testFiles
+  }
+
+  async globTestFiles(filters: string[] = []) {
+    const { include, exclude, includeSource } = this.config
+
+    const testFiles = await this.globFiles(filters, include, exclude)
+
     if (includeSource) {
-      let files = await fg(includeSource, globOptions)
-      if (filters.length)
-        files = files.filter(i => filters.some(f => i.includes(f)))
+      const files = await this.globFiles(filters, includeSource, exclude)
 
       await Promise.all(files.map(async (file) => {
         try {

packages/vitest/src/node/reporters/base.ts

@@ -1,10 +1,10 @@
 import { performance } from 'perf_hooks'
 import c from 'picocolors'
 import type { ErrorWithDiff, File, Reporter, Task, TaskResultPack, UserConsoleLog } from '../../types'
-import { clearInterval, getFullName, getSuites, getTests, getTypecheckTests, hasFailed, hasFailedSnapshot, isNode, relativePath, setInterval } from '../../utils'
+import { clearInterval, getFullName, getSuites, getTests, hasFailed, hasFailedSnapshot, isNode, relativePath, setInterval } from '../../utils'
 import type { Vitest } from '../../node'
 import { F_RIGHT } from '../../utils/figures'
-import { divider, formatProjectName, formatTimeString, getStateString, getStateSymbol, pointer, renderSnapshotSummary } from './renderers/utils'
+import { countTestErrors, divider, formatProjectName, formatTimeString, getStateString, getStateSymbol, pointer, renderSnapshotSummary } from './renderers/utils'
 
 const BADGE_PADDING = '       '
 const HELP_HINT = `${c.dim('press ')}${c.bold('h')}${c.dim(' to show help')}`
@@ -202,7 +202,7 @@ export abstract class BaseReporter implements Reporter {
   }
 
   async reportTestSummary(files: File[]) {
-    const tests = this.mode === 'typecheck' ? getTypecheckTests(files) : getTests(files)
+    const tests = getTests(files)
     const logger = this.ctx.logger
 
     const executionTime = this.end - this.start
@@ -240,9 +240,8 @@ export abstract class BaseReporter implements Reporter {
     logger.log(padTitle('Test Files'), getStateString(files))
     logger.log(padTitle('Tests'), getStateString(tests))
     if (this.mode === 'typecheck') {
-      // has only failed checks
-      const typechecks = getTests(files).filter(t => t.type === 'typecheck')
-      logger.log(padTitle('Type Errors'), getStateString(typechecks, 'errors', false))
+      const failed = tests.filter(t => t.meta?.typecheck && t.result?.errors?.length)
+      logger.log(padTitle('Type Errors'), failed.length ? c.bold(c.red(`${failed} failed`)) : c.dim('no errors'))
     }
     logger.log(padTitle('Start at'), formatTimeString(this._timeStart))
     if (this.watchFilters)
@@ -262,7 +261,7 @@ export abstract class BaseReporter implements Reporter {
 
     const failedSuites = suites.filter(i => i.result?.errors)
     const failedTests = tests.filter(i => i.result?.state === 'fail')
-    const failedTotal = failedSuites.length + failedTests.length
+    const failedTotal = countTestErrors(failedSuites) + countTestErrors(failedTests)
 
     let current = 1
 
@@ -275,8 +274,7 @@ export abstract class BaseReporter implements Reporter {
     }
 
     if (failedTests.length) {
-      const message = this.mode === 'typecheck' ? 'Type Errors' : 'Failed Tests'
-      logger.error(c.red(divider(c.bold(c.inverse(` ${message} ${failedTests.length} `)))))
+      logger.error(c.red(divider(c.bold(c.inverse(` Failed Tests ${failedTests.length} `)))))
       logger.error()
 
       await this.printTaskErrors(failedTests, errorDivider)

packages/vitest/src/node/reporters/renderers/listRenderer.ts

@@ -2,7 +2,7 @@ import c from 'picocolors'
 import cliTruncate from 'cli-truncate'
 import stripAnsi from 'strip-ansi'
 import type { Benchmark, BenchmarkResult, SuiteHooks, Task, VitestRunMode } from '../../../types'
-import { clearInterval, getTests, getTypecheckTests, isTypecheckTest, notNullish, setInterval } from '../../../utils'
+import { clearInterval, getTests, notNullish, setInterval } from '../../../utils'
 import { F_RIGHT } from '../../../utils/figures'
 import type { Logger } from '../../logger'
 import { formatProjectName, getCols, getHookStateSymbol, getStateSymbol } from './utils'
@@ -99,8 +99,8 @@ export function renderTree(tasks: Task[], options: ListRendererOptions, level =
     if (task.type === 'test' && task.result?.retryCount && task.result.retryCount > 1)
       suffix += c.yellow(` (retry x${task.result.retryCount})`)
 
-    if (task.type === 'suite' && !isTypecheckTest(task)) {
-      const tests = options.mode === 'typecheck' ? getTypecheckTests(task) : getTests(task)
+    if (task.type === 'suite' && !task.meta?.typecheck) {
+      const tests = getTests(task)
       suffix += c.dim(` (${tests.length})`)
     }
 

packages/vitest/src/node/reporters/renderers/utils.ts

@@ -91,6 +91,10 @@ export function renderSnapshotSummary(rootDir: string, snapshots: SnapshotSummar
   return summary
 }
 
+export function countTestErrors(tasks: Task[]) {
+  return tasks.reduce((c, i) => c + (i.result?.errors?.length || 0), 0)
+}
+
 export function getStateString(tasks: Task[], name = 'tests', showTotal = true) {
   if (tasks.length === 0)
     return c.dim(`no ${name}`)

packages/vitest/src/runtime/collect.ts

@@ -1,6 +1,6 @@
-import type { File, ResolvedConfig, Suite } from '../types'
+import type { File, ResolvedConfig } from '../types'
 import { getWorkerState, isBrowser, relativePath } from '../utils'
-import { interpretTaskModes, someTasksAreOnly } from '../utils/collect'
+import { calculateSuiteHash, generateHash, interpretTaskModes, someTasksAreOnly } from '../utils/collect'
 import { clearCollectorContext, defaultSuite } from './suite'
 import { getHooks, setHooks } from './map'
 import { processError } from './error'
@@ -9,18 +9,6 @@ import { runSetupFiles } from './setup'
 
 const now = Date.now
 
-function hash(str: string): string {
-  let hash = 0
-  if (str.length === 0)
-    return `${hash}`
-  for (let i = 0; i < str.length; i++) {
-    const char = str.charCodeAt(i)
-    hash = (hash << 5) - hash + char
-    hash = hash & hash // Convert to 32bit integer
-  }
-  return `${hash}`
-}
-
 export async function collectTests(paths: string[], config: ResolvedConfig): Promise<File[]> {
   const files: File[] = []
   const browserHashMap = getWorkerState().browserHashMap!
@@ -37,7 +25,7 @@ export async function collectTests(paths: string[], config: ResolvedConfig): Pro
   for (const filepath of paths) {
     const path = relativePath(config.root, filepath)
     const file: File = {
-      id: hash(path),
+      id: generateHash(path),
       name: path,
       type: 'suite',
       mode: 'run',
@@ -92,7 +80,7 @@ export async function collectTests(paths: string[], config: ResolvedConfig): Pro
         console.error(e)
     }
 
-    calculateHash(file)
+    calculateSuiteHash(file)
 
     const hasOnlyTasks = someTasksAreOnly(file)
     interpretTaskModes(file, config.testNamePattern, hasOnlyTasks, false, config.allowOnly)
@@ -102,11 +90,3 @@ export async function collectTests(paths: string[], config: ResolvedConfig): Pro
 
   return files
 }
-
-function calculateHash(parent: Suite) {
-  parent.tasks.forEach((t, idx) => {
-    t.id = `${parent.id}_${idx}`
-    if (t.type === 'suite')
-      calculateHash(t)
-  })
-}

packages/vitest/src/typecheck/collect.ts

@@ -3,15 +3,19 @@ import { parse as parseAst } from 'acorn'
 import { ancestor as walkAst } from 'acorn-walk'
 import type { RawSourceMap } from 'vite-node'
 
-import type { File, Suite, Vitest } from '../types'
-import { interpretTaskModes, someTasksAreOnly } from '../utils/collect'
-import { TYPECHECK_SUITE } from './constants'
+import type { File, Suite, Test, Vitest } from '../types'
+import { calculateSuiteHash, generateHash, interpretTaskModes, someTasksAreOnly } from '../utils/collect'
 
 interface ParsedFile extends File {
   start: number
   end: number
 }
 
+interface ParsedTest extends Test {
+  start: number
+  end: number
+}
+
 interface ParsedSuite extends Suite {
   start: number
   end: number
@@ -21,9 +25,9 @@ interface LocalCallDefinition {
   start: number
   end: number
   name: string
-  type: string
+  type: 'suite' | 'test'
   mode: 'run' | 'skip' | 'only' | 'todo'
-  task: ParsedSuite | ParsedFile
+  task: ParsedSuite | ParsedFile | ParsedTest
 }
 
 export interface FileInformation {
@@ -42,18 +46,16 @@ export async function collectTests(ctx: Vitest, filepath: string): Promise<null
     ecmaVersion: 'latest',
     allowAwaitOutsideFunction: true,
   })
+  const testFilepath = relative(ctx.config.root, filepath)
   const file: ParsedFile = {
     filepath,
     type: 'suite',
-    id: filepath,
-    name: relative(ctx.config.root, filepath),
+    id: generateHash(testFilepath),
+    name: testFilepath,
     mode: 'run',
     tasks: [],
     start: ast.start,
     end: ast.end,
-    result: {
-      state: 'pass',
-    },
   }
   const definitions: LocalCallDefinition[] = []
   const getName = (callee: any): string | null => {
@@ -80,14 +82,17 @@ export async function collectTests(ctx: Vitest, filepath: string): Promise<null
         return
       const { arguments: [{ value: message }] } = node as any
       const property = callee?.property?.name
-      const mode = !property || property === name ? 'run' : property
-      if (!['run', 'skip', 'todo', 'only'].includes(mode))
+      let mode = !property || property === name ? 'run' : property
+      if (!['run', 'skip', 'todo', 'only', 'skipIf', 'runIf'].includes(mode))
         throw new Error(`${name}.${mode} syntax is not supported when testing types`)
+      // cannot statically analyze, so we always skip it
+      if (mode === 'skipIf' || mode === 'runIf')
+        mode = 'skip'
       definitions.push({
         start: node.start,
         end: node.end,
         name: message,
-        type: name,
+        type: name === 'it' || name === 'test' ? 'test' : 'suite',
         mode,
       } as LocalCallDefinition)
     },
@@ -99,37 +104,49 @@ export async function collectTests(ctx: Vitest, filepath: string): Promise<null
       lastSuite = suite.suite as ParsedSuite
     return lastSuite
   }
-  definitions.sort((a, b) => a.start - b.start).forEach((definition, idx) => {
+  definitions.sort((a, b) => a.start - b.start).forEach((definition) => {
     const latestSuite = updateLatestSuite(definition.start)
     let mode = definition.mode
     if (latestSuite.mode !== 'run') // inherit suite mode, if it's set
       mode = latestSuite.mode
-    const state = mode === 'run' ? 'pass' : mode
-    // expectTypeOf and any type error is actually a "test" ("typecheck"),
-    // and all "test"s should be inside a "suite", so semantics inside typecheck for "test" changes
-    // if we ever allow having multiple errors in a test, we can change type to "test"
-    const task: ParsedSuite = {
-      type: 'suite',
-      id: idx.toString(),
+    if (definition.type === 'suite') {
+      const task: ParsedSuite = {
+        type: definition.type,
+        id: '',
+        suite: latestSuite,
+        file,
+        tasks: [],
+        mode,
+        name: definition.name,
+        end: definition.end,
+        start: definition.start,
+        meta: {
+          typecheck: true,
+        },
+      }
+      definition.task = task
+      latestSuite.tasks.push(task)
+      lastSuite = task
+      return
+    }
+    const task: ParsedTest = {
+      type: definition.type,
+      id: '',
       suite: latestSuite,
       file,
-      tasks: [],
       mode,
+      context: {} as any, // not used in typecheck
       name: definition.name,
       end: definition.end,
       start: definition.start,
-      result: {
-        state,
+      meta: {
+        typecheck: true,
       },
     }
     definition.task = task
     latestSuite.tasks.push(task)
-    if (definition.type === 'describe' || definition.type === 'suite')
-      lastSuite = task
-    else
-      // to show correct amount of "tests" in summary, we mark this with a special symbol
-      Object.defineProperty(task, TYPECHECK_SUITE, { value: true })
   })
+  calculateSuiteHash(file)
   const hasOnly = someTasksAreOnly(file)
   interpretTaskModes(file, ctx.config.testNamePattern, hasOnly, false, ctx.config.allowOnly)
   return {

packages/vitest/src/typecheck/parse.ts

@@ -1,7 +1,7 @@
-import path from 'node:path'
 import url from 'node:url'
 import { writeFile } from 'node:fs/promises'
-import { getTsconfig } from 'get-tsconfig'
+import { join } from 'pathe'
+import { getTsconfig as getTsconfigContent } from 'get-tsconfig'
 import type { TypecheckConfig } from '../types'
 import type { RawErrsMap, TscErrorInfo } from './types'
 
@@ -57,14 +57,14 @@ export async function makeTscErrorInfo(
   ]
 }
 
-export async function getTsconfigPath(root: string, config: TypecheckConfig) {
-  const tempConfigPath = path.join(root, 'tsconfig.temp.json')
+export async function getTsconfig(root: string, config: TypecheckConfig) {
+  const tempConfigPath = join(root, 'tsconfig.temp.json')
 
   const configName = config.tsconfig?.includes('jsconfig.json')
     ? 'jsconfig.json'
     : undefined
 
-  const tsconfig = getTsconfig(config.tsconfig || root, configName)
+  const tsconfig = getTsconfigContent(config.tsconfig || root, configName)
 
   if (!tsconfig)
     throw new Error('no tsconfig.json found')
@@ -75,14 +75,14 @@ export async function getTsconfigPath(root: string, config: TypecheckConfig) {
     tmpTsConfig.compilerOptions = tmpTsConfig.compilerOptions || {}
     tmpTsConfig.compilerOptions.emitDeclarationOnly = false
     tmpTsConfig.compilerOptions.incremental = true
-    tmpTsConfig.compilerOptions.tsBuildInfoFile = path.join(
+    tmpTsConfig.compilerOptions.tsBuildInfoFile = join(
       __dirname,
       'tsconfig.tmp.tsbuildinfo',
     )
 
     const tsconfigFinalContent = JSON.stringify(tmpTsConfig, null, 2)
     await writeFile(tempConfigPath, tsconfigFinalContent)
-    return tempConfigPath
+    return { path: tempConfigPath, config: tmpTsConfig }
   }
   catch (err) {
     throw new Error('failed to write tsconfig.temp.json', { cause: err })