feat: add more options to configure diff output (#2522)

* feat: add more options to configure diff output

* chore: add more tests for diff, cleanup

* docs: add `-` examples

* chore: cleanup

* chore: fix test and wrong "could not display diff"

* test: diff ignores undefined

* Apply suggestions from code review

Co-authored-by: Anjorin Damilare <damilareanjorin1@gmail.com>

* docs: add `--logHeapUsage` to docs

Co-authored-by: Anjorin Damilare <damilareanjorin1@gmail.com>

docs/config/index.md

@@ -347,17 +347,55 @@ Custom reporters for output. Reporters can be [a Reporter instance](https://gith
 ### outputTruncateLength
 
 - **Type:** `number`
-- **Default:** `80`
+- **Default:** `stdout.columns || 80`
+- **CLI:** `--outputTruncateLength <length>`, `--output-truncate-length <length>`
 
-Truncate output diff lines up to `80` number of characters. You may wish to tune this,
-depending on your terminal window width.
+Truncate the size of diff line up to `stdout.columns` or `80` number of characters. You may wish to tune this, depending on your terminal window width. Vitest includes `+-` characters and spaces for this. For example, you might see this diff, if you set this to `6`:
+
+```diff
+// actual line: "Text that seems correct"
+- Text...
++ Test...
+```
 
 ### outputDiffLines
 
 - **Type:** `number`
 - **Default:** `15`
+- **CLI:** `--outputDiffLines <lines>`, `--output-diff-lines <lines>`
+
+Limit the number of single output diff lines up to `15`. Vitest counts all `+-` lines when determining when to stop. For example, you might see diff like this, if you set this property to `3`:
+
+```diff
+- test: 1,
++ test: 2,
+- obj: '1',
+...
+- test2: 1,
++ test2: 1,
+- obj2: '2',
+...
+```
+
+### outputDiffMaxLines
+
+- **Type:** `number`
+- **Default:** `50`
+- **CLI:** `--outputDiffMaxLines <lines>`, `--output-diff-max-lines <lines>`
+- **Version:** Since Vitest 0.26.0
+
+The maximum number of lines to display in diff window. Beware that if you have a large object with many small diffs, you might not see all of them at once.
+
+### outputDiffMaxSize
+
+- **Type:** `number`
+- **Default:** `10000`
+- **CLI:** `--outputDiffMaxSize <length>`, `--output-diff-max-size <length>`
+- **Version:** Since Vitest 0.26.0
+
+The maximum length of the stringified object before the diff happens. Vitest tries to stringify an object before doing a diff, but if the object is too large, it will reduce the depth of the object to fit within this limit. Because of this, if the object is too big or nested, you might not see the diff.
 
-Limit number of output diff lines up to `15`.
+Increasing this limit can increase the duration of diffing.
 
 ### outputFile
 

docs/guide/cli.md

@@ -58,6 +58,8 @@ vitest related /src/index.ts /src/hello-world.js
 | `--silent` | Silent console output from tests |
 | `--isolate` | Isolate environment for each test file (default: `true`) |
 | `--reporter <name>` | Select reporter: `default`, `verbose`, `dot`, `junit`, `json`, or a path to a custom reporter |
+| `--outputDiffMaxSize <length>` | Object diff output max size (default: 10000) |
+| `--outputDiffMaxLines <lines>` | Max lines in diff output window (default: 50) |
 | `--outputTruncateLength <length>` | Truncate output diff lines up to `<length>` number of characters. |
 | `--outputDiffLines <lines>` | Limit number of output diff lines up to `<lines>`. |
 | `--outputFile <filename/-s>` | Write test results to a file when the `--reporter=json` or `--reporter=junit` option is also specified <br /> Via [cac's dot notation] you can specify individual outputs for multiple reporters |
@@ -70,6 +72,7 @@ vitest related /src/index.ts /src/hello-world.js
 | `--browser` | Run tests in browser |
 | `--environment <env>` | Runner environment (default: `node`) |
 | `--passWithNoTests` | Pass when no tests found |
+| `--logHeapUsage` | Show the size of heap for each test |
 | `--allowOnly` | Allow tests and suites that are marked as `only` (default: false in CI, true otherwise) |
 | `--dangerouslyIgnoreUnhandledErrors` | Ignore any unhandled errors that occur |
 | `--changed [since]` | Run tests that are affected by the changed files (default: false). See [docs](#changed) |

packages/vitest/src/integrations/chai/jest-matcher-utils.ts

@@ -102,8 +102,8 @@ const SPACE_SYMBOL = '\u{00B7}' // middle dot
 const replaceTrailingSpaces = (text: string): string =>
   text.replace(/\s+$/gm, spaces => SPACE_SYMBOL.repeat(spaces.length))
 
-export function stringify(object: unknown, maxDepth = 10, options?: PrettyFormatOptions): string {
-  const MAX_LENGTH = 10000
+export function stringify(object: unknown, maxDepth = 10, { maxLength, ...options }: PrettyFormatOptions & { maxLength?: number } = {}): string {
+  const MAX_LENGTH = maxLength ?? 10000
   let result
 
   try {

packages/vitest/src/node/cli.ts

@@ -24,8 +24,10 @@ cli
   .option('--silent', 'silent console output from tests')
   .option('--isolate', 'isolate environment for each test file (default: true)')
   .option('--reporter <name>', 'reporter')
-  .option('--outputTruncateLength <length>', 'diff output length (default: 80)')
-  .option('--outputDiffLines <lines>', 'number of diff output lines (default: 15)')
+  .option('--outputDiffMaxSize <length>', 'object diff output max size (default: 10000)')
+  .option('--outputDiffMaxLines <length>', 'max lines in diff output window (default: 50)')
+  .option('--outputTruncateLength <length>', 'diff output line length (default: 80)')
+  .option('--outputDiffLines <lines>', 'number of lines in single diff (default: 15)')
   .option('--outputFile <filename/-s>', 'write test results to a file when the --reporter=json or --reporter=junit option is also specified, use cac\'s dot notation for individual outputs of multiple reporters')
   .option('--coverage', 'enable coverage report')
   .option('--run', 'do not watch')
@@ -35,6 +37,7 @@ cli
   .option('--browser', 'run tests in browser')
   .option('--environment <env>', 'runner environment (default: node)')
   .option('--passWithNoTests', 'pass when no tests found')
+  .option('--logHeapUsage', 'show the size of heap for each test')
   .option('--allowOnly', 'Allow tests and suites that are marked as only (default: !process.env.CI)')
   .option('--dangerouslyIgnoreUnhandledErrors', 'Ignore any unhandled errors that occur')
   .option('--shard <shard>', 'Test suite shard to execute in a format of <index>/<count>')

packages/vitest/src/node/error.ts

@@ -160,8 +160,14 @@ function handleImportOutsideModuleError(stack: string, ctx: Vitest) {
 }\n`)))
 }
 
-function displayDiff(actual: string, expected: string, console: Console, options?: Omit<DiffOptions, 'showLegend'>) {
-  console.error(c.gray(unifiedDiff(actual, expected, options)) + '\n')
+export function displayDiff(actual: string, expected: string, console: Console, options: Omit<DiffOptions, 'showLegend'> = {}) {
+  const diff = unifiedDiff(actual, expected, options)
+  const dim = options.noColor ? (s: string) => s : c.dim
+  const black = options.noColor ? (s: string) => s : c.black
+  if (diff)
+    console.error(diff + '\n')
+  else if (actual && expected && actual !== '"undefined"' && expected !== '"undefined"')
+    console.error(dim('Could not display diff. It\'s possible objects are too large to compare.\nTry increasing ') + black('--outputDiffMaxSize') + dim(' option.\n'))
 }
 
 function printErrorMessage(error: ErrorWithDiff, logger: Logger) {

packages/vitest/src/runtime/error.ts

@@ -1,7 +1,7 @@
 import util from 'util'
 import { util as ChaiUtil } from 'chai'
 import { stringify } from '../integrations/chai/jest-matcher-utils'
-import { deepClone, getType } from '../utils'
+import { deepClone, getType, getWorkerState } from '../utils'
 
 const IS_RECORD_SYMBOL = '@@__IMMUTABLE_RECORD__@@'
 const IS_COLLECTION_SYMBOL = '@@__IMMUTABLE_ITERABLE__@@'
@@ -102,10 +102,13 @@ export function processError(err: any) {
   err.actual = replacedActual
   err.expected = replacedExpected
 
+  const workerState = getWorkerState()
+  const maxDiffSize = workerState.config.outputDiffMaxSize
+
   if (typeof err.expected !== 'string')
-    err.expected = stringify(err.expected)
+    err.expected = stringify(err.expected, 10, { maxLength: maxDiffSize })
   if (typeof err.actual !== 'string')
-    err.actual = stringify(err.actual)
+    err.actual = stringify(err.actual, 10, { maxLength: maxDiffSize })
 
   // some Error implementations don't allow rewriting message
   try {

packages/vitest/src/typecheck/parse.ts

@@ -72,7 +72,7 @@ export async function getTsconfigPath(root: string, config: TypecheckConfig) {
   try {
     const tmpTsConfig: Record<string, any> = { ...tsconfig.config }
 
-    tmpTsConfig.compilerOptions ??= {}
+    tmpTsConfig.compilerOptions = tmpTsConfig.compilerOptions || {}
     tmpTsConfig.compilerOptions.emitDeclarationOnly = false
     tmpTsConfig.compilerOptions.incremental = true
     tmpTsConfig.compilerOptions.tsBuildInfoFile = path.join(

packages/vitest/src/types/config.ts

@@ -156,20 +156,37 @@ export interface InlineConfig {
 
   /**
    * Custom reporter for output. Can contain one or more built-in report names, reporter instances,
-   * and/or paths to custom reporters
+   * and/or paths to custom reporters.
    */
   reporters?: Arrayable<BuiltinReporters | 'html' | Reporter | Omit<string, BuiltinReporters>>
 
   /**
-   * diff output length
+   * Truncates lines in the output to the given length.
+   * @default stdout.columns || 80
    */
   outputTruncateLength?: number
 
   /**
-   * number of diff output lines
+   * Maximum number of line to show in a single diff.
+   * @default 15
    */
   outputDiffLines?: number
 
+  /**
+   * The maximum number of characters allowed in a single object before doing a diff.
+   * Vitest tries to stringify an object before doing a diff, but if the object is too large,
+   * it will reduce the depth of the object to fit within this limit.
+   * Because of this if object is too big or nested, you might not see the diff.
+   * @default 10000
+   */
+  outputDiffMaxSize?: number
+
+  /**
+   * Maximum number of lines in a diff overall.
+   * @default 50
+   */
+  outputDiffMaxLines?: number
+
   /**
    * Write test results to a file when the --reporter=json` or `--reporter=junit` option is also specified.
    * Also definable individually per reporter by using an object instead.

packages/vitest/src/utils/diff.ts

@@ -7,6 +7,8 @@ export function formatLine(line: string, outputTruncateLength?: number) {
 }
 
 export interface DiffOptions {
+  noColor?: boolean
+  outputDiffMaxLines?: number
   outputTruncateLength?: number
   outputDiffLines?: number
   showLegend?: boolean
@@ -25,17 +27,23 @@ export function unifiedDiff(actual: string, expected: string, options: DiffOptio
   if (actual === expected)
     return ''
 
-  const { outputTruncateLength, outputDiffLines, showLegend = true } = options
+  const { outputTruncateLength, outputDiffLines, outputDiffMaxLines, noColor, showLegend = true } = options
 
   const indent = '  '
   const diffLimit = outputDiffLines || 15
+  const diffMaxLines = outputDiffMaxLines || 50
 
   const counts = {
     '+': 0,
     '-': 0,
   }
   let previousState: '-' | '+' | null = null
   let previousCount = 0
+
+  const str = (str: string) => str
+  const dim = noColor ? str : c.dim
+  const green = noColor ? str : c.green
+  const red = noColor ? str : c.red
   function preprocess(line: string) {
     if (!line || line.match(/\\ No newline/))
       return
@@ -49,42 +57,55 @@ export function unifiedDiff(actual: string, expected: string, options: DiffOptio
       previousCount++
       counts[char]++
       if (previousCount === diffLimit)
-        return c.dim(`${char} ...`)
+        return dim(`${char} ...`)
       else if (previousCount > diffLimit)
         return
     }
     return line
   }
 
   const msg = diff.createPatch('string', expected, actual)
-  const lines = msg.split('\n').slice(5).map(preprocess).filter(Boolean) as string[]
+  let lines = msg.split('\n').slice(5).map(preprocess).filter(Boolean) as string[]
+  let moreLines = 0
   const isCompact = counts['+'] === 1 && counts['-'] === 1 && lines.length === 2
 
+  if (lines.length > diffMaxLines) {
+    const firstDiff = lines.findIndex(line => line[0] === '-' || line[0] === '+')
+    const displayLines = lines.slice(firstDiff - 2, diffMaxLines)
+    const lastDisplayedIndex = firstDiff - 2 + diffMaxLines
+    if (lastDisplayedIndex < lines.length)
+      moreLines = lines.length - lastDisplayedIndex
+    lines = displayLines
+  }
+
   let formatted = lines.map((line: string) => {
     line = line.replace(/\\"/g, '"')
     if (line[0] === '-') {
       line = formatLine(line.slice(1), outputTruncateLength)
       if (isCompact)
-        return c.green(line)
-      return c.green(`- ${formatLine(line, outputTruncateLength)}`)
+        return green(line)
+      return green(`- ${formatLine(line, outputTruncateLength)}`)
     }
     if (line[0] === '+') {
       line = formatLine(line.slice(1), outputTruncateLength)
       if (isCompact)
-        return c.red(line)
-      return c.red(`+ ${formatLine(line, outputTruncateLength)}`)
+        return red(line)
+      return red(`+ ${formatLine(line, outputTruncateLength)}`)
     }
     if (line.match(/@@/))
       return '--'
     return ` ${line}`
   })
 
+  if (moreLines)
+    formatted.push(dim(`... ${moreLines} more lines`))
+
   if (showLegend) {
     // Compact mode
     if (isCompact) {
       formatted = [
-        `${c.green('- Expected')}   ${formatted[0]}`,
-        `${c.red('+ Received')}   ${formatted[1]}`,
+        `${green('- Expected')}   ${formatted[0]}`,
+        `${red('+ Received')}   ${formatted[1]}`,
       ]
     }
     else {
@@ -96,12 +117,12 @@ export function unifiedDiff(actual: string, expected: string, options: DiffOptio
         formatted[last] = formatted[last].slice(0, formatted[last].length - 1)
 
       formatted.unshift(
-        c.green(`- Expected  - ${counts['-']}`),
-        c.red(`+ Received  + ${counts['+']}`),
+        green(`- Expected  - ${counts['-']}`),
+        red(`+ Received  + ${counts['+']}`),
         '',
       )
     }
   }
 
-  return formatted.map(i => indent + i).join('\n')
+  return formatted.map(i => i ? (indent + i) : i).join('\n')
 }

test/core/test/diff.test.ts

@@ -0,0 +1,81 @@
+import { expect, test, vi } from 'vitest'
+import { displayDiff } from 'vitest/src/node/error'
+import { stringify } from 'vitest/src/integrations/chai/jest-matcher-utils'
+
+test('displays an error for large objects', () => {
+  const objectA = new Array(1000).fill(0).map((_, i) => ({ i, long: 'a'.repeat(i) }))
+  const objectB = new Array(1000).fill(0).map((_, i) => ({ i, long: 'b'.repeat(i) }))
+  const console = { log: vi.fn(), error: vi.fn() }
+  displayDiff(stringify(objectA), stringify(objectB), console as any, { noColor: true })
+  expect(console.error.mock.calls[0][0]).toMatchInlineSnapshot(`
+    "Could not display diff. It's possible objects are too large to compare.
+    Try increasing --outputDiffMaxSize option.
+    "
+  `)
+})
+
+test('displays an error for large objects', () => {
+  const console = { log: vi.fn(), error: vi.fn() }
+  displayDiff(stringify('undefined'), stringify('undefined'), console as any, { noColor: true })
+  expect(console.error).not.toHaveBeenCalled()
+})
+
+test('displays diff', () => {
+  const objectA = { a: 1, b: 2 }
+  const objectB = { a: 1, b: 3 }
+  const console = { log: vi.fn(), error: vi.fn() }
+  displayDiff(stringify(objectA), stringify(objectB), console as any, { noColor: true })
+  expect(console.error.mock.calls[0][0]).toMatchInlineSnapshot(`
+    "  - Expected  - 1
+      + Received  + 1
+
+        Object {
+          \\"a\\": 1,
+      -   \\"b\\": 3,
+      +   \\"b\\": 2,
+        }
+    "
+  `)
+})
+
+test('displays long diff', () => {
+  const objectA = { a: 1, b: 2, d: 4, e: 5, f: 6, g: 7, h: 8, i: 9, j: 10, k: 11, l: 12, m: 13, n: 14, o: 15, p: 16, q: 17, r: 18, s: 19, t: 20, u: 21, v: 22, w: 23, x: 24, y: 25, z: 26 }
+  const objectB = { a: 1, b: 3, k: 11, l: 12, m: 13, n: 14, p: 16, o: 17, r: 18, s: 23, t: 88, u: 21, v: 44, w: 23, x: 24, y: 25, z: 26 }
+  const console = { log: vi.fn(), error: vi.fn() }
+  displayDiff(stringify(objectA), stringify(objectB), console as any, { noColor: true, outputDiffMaxLines: 5 })
+  expect(console.error.mock.calls[0][0]).toMatchInlineSnapshot(`
+    "  - Expected  - 5
+      + Received  + 13
+
+        Object {
+          \\"a\\": 1,
+      -   \\"b\\": 3,
+      +   \\"b\\": 2,
+      +   \\"d\\": 4,
+      ... 26 more lines
+    "
+  `)
+})
+
+test('displays truncated diff', () => {
+  const stringA = `Lorem ipsum dolor sit amet, consectetur adipiscing elit.
+Suspendisse viverra sapien ac venenatis lacinia.
+Morbi consectetur arcu nec lorem lacinia tempus.`
+  const objectB = `Quisque hendrerit metus id dapibus pulvinar.
+Quisque pellentesque enim a elit faucibus cursus.
+Sed in tellus aliquet mauris interdum semper a in lacus.`
+  const console = { log: vi.fn(), error: vi.fn() }
+  displayDiff((stringA), (objectB), console as any, { noColor: true, outputTruncateLength: 14 })
+  expect(console.error.mock.calls[0][0]).toMatchInlineSnapshot(`
+    "  - Expected  - 3
+      + Received  + 3
+
+      - Quisque h…
+      - Quisque p…
+      - Sed in te…
+      + Lorem ips…
+      + Suspendis…
+      + Morbi con…
+    "
+  `)
+})