chore: Reduce flakiness of the tests (#15860)

* chore: Reduce flakiness of the tests * wait for logs in batch tx tests * Increase timeout and retry couple of tests * Force every schema hash to be unique
prisma · Oct 19, 2022 · 89ef63e · 89ef63e
1 parent c1afd29
commit 89ef63e
Show file tree

Hide file tree

Showing 8 changed files with 53 additions and 7 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -338,6 +338,7 @@ jobs:
           TEST_FUNCTIONAL_MSSQL_URI: 'sqlserver://localhost:1433;database=PRISMA_DB_NAME;user=SA;password=Pr1sm4_Pr1sm4;trustServerCertificate=true;'
           TEST_FUNCTIONAL_MONGO_URI: 'mongodb://root:prisma@localhost:27018/PRISMA_DB_NAME?authSource=admin'
           TEST_FUNCTIONAL_COCKROACH_URI: 'postgresql://prisma@localhost:26257/PRISMA_DB_NAME'
+          NODE_OPTIONS: '--max-old-space-size=8096'
 
       - run: pnpm run test:functional --data-proxy --edge-client
         working-directory: packages/client
@@ -350,6 +351,7 @@ jobs:
           TEST_FUNCTIONAL_MSSQL_URI: 'sqlserver://localhost:1433;database=PRISMA_DB_NAME;user=SA;password=Pr1sm4_Pr1sm4;trustServerCertificate=true;'
           TEST_FUNCTIONAL_MONGO_URI: 'mongodb://root:prisma@localhost:27018/PRISMA_DB_NAME?authSource=admin'
           TEST_FUNCTIONAL_COCKROACH_URI: 'postgresql://prisma@localhost:26257/PRISMA_DB_NAME'
+          NODE_OPTIONS: '--max-old-space-size=8096'
 
       - uses: codecov/codecov-action@v3
         with:

diff --git a/packages/client/tests/functional/_example/tests.ts b/packages/client/tests/functional/_example/tests.ts
@@ -49,8 +49,8 @@ testMatrix.setupTestSuite(
       expect(suiteMeta.testFileName).toEqual(path.basename(__filename))
     })
 
-    test('getTestSuiteSchema', async () => {
-      const schemaString = await getTestSuiteSchema(suiteMeta, suiteConfig)
+    test('getTestSuiteSchema', () => {
+      const schemaString = getTestSuiteSchema(suiteMeta, suiteConfig)
 
       expect(schemaString).toContain('generator')
       expect(schemaString).toContain('datasource')

diff --git a/packages/client/tests/functional/_utils/getTestSuiteInfo.ts b/packages/client/tests/functional/_utils/getTestSuiteInfo.ts
@@ -143,7 +143,15 @@ function getTestSuiteParametersString(configs: Record<string, string>[]) {
  * @returns
  */
 export function getTestSuiteSchema(suiteMeta: TestSuiteMeta, matrixOptions: Record<string, string>) {
-  return require(suiteMeta._schemaPath).default(matrixOptions)
+  const schemaStr = require(suiteMeta._schemaPath).default(matrixOptions)
+
+  // By default, mini-proxy distiguishes different engine instances using inline schema hash
+  // In case 2 tests are running in parallel with identical schema, this can cause all kinds of problems
+  // Adding a unique comment at the top of schema file forces them to have different hash and avoids
+  // those problems
+  const header = `// ${JSON.stringify({ test: suiteMeta.testPath, matrixOptions })}`
+
+  return `${header}\n${schemaStr}`
 }
 
 /**

diff --git a/packages/client/tests/functional/_utils/setupTestSuiteClient.ts b/packages/client/tests/functional/_utils/setupTestSuiteClient.ts
@@ -35,7 +35,7 @@ export async function setupTestSuiteClient({
 }) {
   const suiteFolderPath = getTestSuiteFolderPath(suiteMeta, suiteConfig)
   const previewFeatures = getTestSuitePreviewFeatures(suiteConfig.matrixOptions)
-  const schema = await getTestSuiteSchema(suiteMeta, suiteConfig.matrixOptions)
+  const schema = getTestSuiteSchema(suiteMeta, suiteConfig.matrixOptions)
   const dmmf = await getDMMF({ datamodel: schema, previewFeatures })
   const config = await getConfig({ datamodel: schema, ignoreEnvVarErrors: true })
   const generator = config.generators.find((g) => parseEnvValue(g.provider) === 'prisma-client-js')

diff --git a/packages/client/tests/functional/_utils/setupTestSuiteMatrix.ts b/packages/client/tests/functional/_utils/setupTestSuiteMatrix.ts
@@ -128,7 +128,7 @@ function setupTestSuiteMatrix(
         delete globalThis['prisma']
         delete globalThis['Prisma']
         delete globalThis['newPrismaClient']
-      })
+      }, 120_000)
 
       tests(suiteConfig.matrixOptions, suiteMeta, clientMeta)
     })

diff --git a/packages/client/tests/functional/_utils/waitFor.ts b/packages/client/tests/functional/_utils/waitFor.ts
@@ -0,0 +1,32 @@
+import { performance } from 'perf_hooks'
+
+const MAX_WAIT = 5000
+
+/**
+ * Waits until provided callback stops throwing.
+ * If it does not happen within MAX_WAIT, throws last thrown error
+ *
+ * Simplified version of testing-library's waitFor that works without DOM
+ * https://testing-library.com/docs/dom-testing-library/api-async/#waitfor
+ * @param cb
+ */
+export async function waitFor(cb: () => void | Promise<void>) {
+  const start = performance.now()
+  let error: unknown = null
+
+  while (performance.now() - start < MAX_WAIT) {
+    try {
+      await cb()
+      return
+    } catch (e) {
+      error = e
+      await delay(100)
+    }
+  }
+
+  throw error
+}
+
+function delay(ms: number) {
+  return new Promise((resolve) => setTimeout(resolve, ms))
+}
diff --git a/packages/client/tests/functional/batch-transaction-isolation-level/tests.ts b/packages/client/tests/functional/batch-transaction-isolation-level/tests.ts
@@ -1,4 +1,5 @@
 import { NewPrismaClient } from '../_utils/types'
+import { waitFor } from '../_utils/waitFor'
 import testMatrix from './_matrix'
 // @ts-ignore
 import type { Prisma as PrismaNamespace, PrismaClient } from './node_modules/@prisma/client'
@@ -39,8 +40,10 @@ testMatrix.setupTestSuite(
           isolationLevel: level(),
         })
 
-        // eslint-disable-next-line jest/no-standalone-expect
-        expect(queries).toContain(expectSql)
+        await waitFor(() => {
+          // eslint-disable-next-line jest/no-standalone-expect
+          expect(queries).toContain(expectSql)
+        })
       })
     }
 

diff --git a/packages/fetch-engine/src/__tests__/download.test.ts b/packages/fetch-engine/src/__tests__/download.test.ts
@@ -16,6 +16,7 @@ const dirname = process.platform === 'win32' ? __dirname.split(path.sep).join('/
 
 // Network can be slow, especially for macOS in CI.
 jest.setTimeout(300_000)
+jest.retryTimes(3)
 
 describe('download', () => {
   beforeEach(async () => {