test: stable tests by forcing order

- Adds work-around for vitest-dev/vitest#1619
AriPerkkio · Jul 9, 2022 · 1159fbf · 1159fbf
1 parent 025d64e
commit 1159fbf
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 27 deletions.
diff --git a/test/index.test.ts b/test/index.test.ts
@@ -33,15 +33,15 @@ test('writes a report', () => {
             <failure message=\\"expected false to be true // Object.is equality\\">
               <![CDATA[AssertionError: expected false to be true // Object.is equality
           at <process-cwd>/test/fixtures/animals.test.ts:15:47
-          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.46442ffe.js
-          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.46442ffe.js
-          at runTest (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runFiles (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async startTests (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async <process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js]]>
+          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.6d23d202.mjs:1938:36
+          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.6d23d202.mjs:1826:26
+          at runTest (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:487:40)
+          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:570:13)
+          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:570:13)
+          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:570:13)
+          at async runFiles (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:610:5)
+          at async startTests (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:617:3)
+          at async <process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:661:9]]>
             </failure>
           </testCase>
           <testCase name=\\"animals - flying ones - bird can fly\\" duration=\\"123\\" />
@@ -54,30 +54,30 @@ test('writes a report', () => {
             <failure message=\\"expected 4 to deeply equal 8\\">
               <![CDATA[AssertionError: expected 4 to deeply equal 8
           at <process-cwd>/test/fixtures/math.test.ts:15:42
-          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.46442ffe.js
-          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.46442ffe.js
-          at runTest (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runFiles (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async startTests (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async <process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js
-          at async withEnv (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)]]>
+          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.6d23d202.mjs:1938:36
+          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.6d23d202.mjs:1826:26
+          at runTest (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:487:40)
+          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:570:13)
+          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:570:13)
+          at async runFiles (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:610:5)
+          at async startTests (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:617:3)
+          at async <process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:661:9
+          at async withEnv (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:171:5)]]>
             </failure>
           </testCase>
           <testCase name=\\"math - complex calculation\\" duration=\\"123\\">
             <error message=\\"16.divideByTwo is not a function\\">
               <![CDATA[TypeError: 16.divideByTwo is not a function
           at <process-cwd>/test/fixtures/math.test.ts:18:38
-          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.46442ffe.js
-          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.46442ffe.js
-          at runTest (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async runFiles (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async startTests (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)
-          at async <process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js
-          at async withEnv (<process-cwd>/node_modules/vitest/dist/vendor-entry.225630f2.js)]]>
+          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.6d23d202.mjs:1938:36
+          at <process-cwd>/node_modules/vitest/dist/chunk-runtime-chain.6d23d202.mjs:1826:26
+          at runTest (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:487:40)
+          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:570:13)
+          at async runSuite (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:570:13)
+          at async runFiles (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:610:5)
+          at async startTests (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:617:3)
+          at async <process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:661:9
+          at async withEnv (<process-cwd>/node_modules/vitest/dist/vendor-entry.1ad8a08d.mjs:171:5)]]>
             </error>
           </testCase>
           <testCase name=\\"math - random numbers are unstable\\">

diff --git a/test/vite.test-config.ts b/test/vite.test-config.ts
@@ -1,4 +1,5 @@
 import { defineConfig } from 'vitest/config';
+import { BaseSequencer } from 'vitest/node';
 
 import SonarReporter from '../src/sonar-reporter';
 
@@ -13,5 +14,21 @@ export default defineConfig({
             'test/fixtures/animals.test.ts',
             'test/fixtures/math.test.ts',
         ],
+
+        // Run tests in fixed order to keep generated XML report stable between runs
+        sequence: {
+            // @ts-expect-error -- vitest-dev/vitest#1619
+            sequencer: class Seqencer extends BaseSequencer {
+                async sort(files: string[]) {
+                    const sorted = files.sort();
+
+                    return sorted;
+                }
+
+                async shard(files: string[]) {
+                    return files;
+                }
+            },
+        },
     },
 });
diff --git a/vite.config.ts b/vite.config.ts
@@ -4,5 +4,6 @@ export default defineConfig({
     test: {
         reporters: 'verbose',
         include: ['test/*.test.ts'],
+        watchExclude: ['report-from-tests.xml'],
     },
 });