fix: added transaction retry logic in cockroachdb (#10032)

* added transaction retry logic in cockroachdb

* added option to control max transaction retries;
added delay before transaction retry;
updated docs;

* fixes in retry logic

* enable storing queries after retrying transaction

docs/data-source-options.md

@@ -182,15 +182,17 @@ Different RDBMS-es have their own specific options.
 
 -   `poolErrorHandler` - A function that get's called when underlying pool emits `'error'` event. Takes single parameter (error instance) and defaults to logging with `warn` level.
 
+-   `maxTransactionRetries` - A maximum number of transaction retries in case of 40001 error. Defaults to 5.
+
 -   `logNotifications` - A boolean to determine whether postgres server [notice messages](https://www.postgresql.org/docs/current/plpgsql-errors-and-messages.html) and [notification events](https://www.postgresql.org/docs/current/sql-notify.html) should be included in client's logs with `info` level (default: `false`).
 
 -   `installExtensions` - A boolean to control whether to install necessary postgres extensions automatically or not (default: `true`)
 
 -   `applicationName` - A string visible in statistics and logs to help referencing an application to a connection (default: `undefined`)
 
--   `parseInt8` - A boolean to enable parsing 64-bit integers (int8) as JavaScript integers.  
-    By default int8 (bigint) values are returned as strings to avoid overflows.  
-    JavaScript doesn't have support for 64-bit integers, the maximum safe integer in js is: Number.MAX_SAFE_INTEGER (`+2^53`). Be careful when enabling `parseInt8`.  
+-   `parseInt8` - A boolean to enable parsing 64-bit integers (int8) as JavaScript integers.
+    By default int8 (bigint) values are returned as strings to avoid overflows.
+    JavaScript doesn't have support for 64-bit integers, the maximum safe integer in js is: Number.MAX_SAFE_INTEGER (`+2^53`). Be careful when enabling `parseInt8`.
     Note: This option is ignored if the undelying driver does not support it.
 
 ## `sqlite` data source options

src/driver/cockroachdb/CockroachConnectionOptions.ts

@@ -56,9 +56,14 @@ export interface CockroachConnectionOptions
      */
     readonly applicationName?: string
 
-    /*
+    /**
      * Function handling errors thrown by drivers pool.
      * Defaults to logging error with `warn` level.
      */
     readonly poolErrorHandler?: (err: any) => any
+
+    /**
+     * Max number of transaction retries in case of 40001 error.
+     */
+    readonly maxTransactionRetries?: number
 }

src/driver/cockroachdb/CockroachQueryRunner.ts

@@ -67,6 +67,11 @@ export class CockroachQueryRunner
      */
     protected storeQueries: boolean = false
 
+    /**
+     * Current number of transaction retries in case of 40001 error.
+     */
+    protected transactionRetries: number = 0
+
     // -------------------------------------------------------------------------
     // Constructor
     // -------------------------------------------------------------------------
@@ -141,6 +146,7 @@ export class CockroachQueryRunner
      */
     async startTransaction(isolationLevel?: IsolationLevel): Promise<void> {
         this.isTransactionActive = true
+        this.transactionRetries = 0
         try {
             await this.broadcaster.broadcast("BeforeTransactionStart")
         } catch (err) {
@@ -182,21 +188,12 @@ export class CockroachQueryRunner
             this.transactionDepth -= 1
         } else {
             this.storeQueries = false
-            try {
-                await this.query("RELEASE SAVEPOINT cockroach_restart")
-                await this.query("COMMIT")
-                this.queries = []
-                this.isTransactionActive = false
-                this.transactionDepth -= 1
-            } catch (e) {
-                if (e.code === "40001") {
-                    await this.query("ROLLBACK TO SAVEPOINT cockroach_restart")
-                    for (const q of this.queries) {
-                        await this.query(q.query, q.parameters)
-                    }
-                    await this.commitTransaction()
-                }
-            }
+            await this.query("RELEASE SAVEPOINT cockroach_restart")
+            await this.query("COMMIT")
+            this.queries = []
+            this.isTransactionActive = false
+            this.transactionRetries = 0
+            this.transactionDepth -= 1
         }
 
         await this.broadcaster.broadcast("AfterTransactionCommit")
@@ -220,6 +217,7 @@ export class CockroachQueryRunner
             await this.query("ROLLBACK")
             this.queries = []
             this.isTransactionActive = false
+            this.transactionRetries = 0
         }
         this.transactionDepth -= 1
 
@@ -295,16 +293,45 @@ export class CockroachQueryRunner
                 return result.raw
             }
         } catch (err) {
-            if (err.code !== "40001") {
+            if (
+                err.code === "40001" &&
+                this.isTransactionActive &&
+                this.transactionRetries <
+                    (this.driver.options.maxTransactionRetries || 5)
+            ) {
+                this.transactionRetries += 1
+                this.storeQueries = false
+                await this.query("ROLLBACK TO SAVEPOINT cockroach_restart")
+                const sleepTime =
+                    2 ** this.transactionRetries *
+                    0.1 *
+                    (Math.random() + 0.5) *
+                    1000
+                await new Promise((resolve) => setTimeout(resolve, sleepTime))
+
+                let result = undefined
+                for (const q of this.queries) {
+                    this.driver.connection.logger.logQuery(
+                        `Retrying transaction for query "${q.query}"`,
+                        q.parameters,
+                        this,
+                    )
+                    result = await this.query(q.query, q.parameters)
+                }
+                this.transactionRetries = 0
+                this.storeQueries = true
+
+                return result
+            } else {
                 this.driver.connection.logger.logQueryError(
                     err,
                     query,
                     parameters,
                     this,
                 )
-            }
 
-            throw new QueryFailedError(query, parameters, err)
+                throw new QueryFailedError(query, parameters, err)
+            }
         }
     }
 

test/github-issues/9984/entity/Post.ts

@@ -0,0 +1,12 @@
+import { Entity } from "../../../../src/decorator/entity/Entity"
+import { Column } from "../../../../src/decorator/columns/Column"
+import { PrimaryGeneratedColumn } from "../../../../src/decorator/columns/PrimaryGeneratedColumn"
+
+@Entity()
+export class Post {
+    @PrimaryGeneratedColumn()
+    id: number
+
+    @Column()
+    name: string
+}

test/github-issues/9984/issue-9984.ts

@@ -0,0 +1,76 @@
+import { DataSource } from "../../../src"
+import {
+    closeTestingConnections,
+    createTestingConnections,
+    reloadTestingDatabases,
+} from "../../utils/test-utils"
+import { Post } from "./entity/Post.js"
+import { expect } from "chai"
+
+describe("github issues > #9984 TransactionRetryWithProtoRefreshError should be handled by TypeORM", () => {
+    let dataSources: DataSource[]
+
+    before(async () => {
+        dataSources = await createTestingConnections({
+            entities: [Post],
+            enabledDrivers: ["cockroachdb"],
+        })
+    })
+
+    beforeEach(() => reloadTestingDatabases(dataSources))
+    after(() => closeTestingConnections(dataSources))
+
+    it("should retry transaction on 40001 error with 'inject_retry_errors_enabled=true'", () =>
+        Promise.all(
+            dataSources.map(async (dataSource) => {
+                const queryRunner = dataSource.createQueryRunner()
+                await dataSource.query("SET inject_retry_errors_enabled = true")
+                await queryRunner.startTransaction()
+
+                const post = new Post()
+                post.name = `post`
+                await queryRunner.manager.save(post)
+                await queryRunner.commitTransaction()
+                await queryRunner.release()
+                await dataSource.query(
+                    "SET inject_retry_errors_enabled = false",
+                )
+                const loadedPost = await dataSource.manager.findOneBy(Post, {
+                    id: post.id,
+                })
+                expect(loadedPost).to.be.not.undefined
+            }),
+        ))
+
+    it("should retry transaction on 40001 error", () =>
+        Promise.all(
+            dataSources.map(async (dataSource) => {
+                const queryRunner = dataSource.createQueryRunner()
+                const post = new Post()
+                post.name = "post"
+                await queryRunner.manager.save(post)
+                await queryRunner.release()
+
+                const query = async (name: string) => {
+                    const queryRunner = dataSource.createQueryRunner()
+                    await queryRunner.startTransaction()
+                    const updatedPost = new Post()
+                    updatedPost.id = post.id
+                    updatedPost.name = name
+                    await queryRunner.manager.save(updatedPost)
+                    await queryRunner.commitTransaction()
+                    await queryRunner.release()
+                }
+
+                await Promise.all([1, 2, 3].map((i) => query(`changed_${i}`)))
+
+                const loadedPost = await dataSource.manager.findOneByOrFail(
+                    Post,
+                    {
+                        id: post.id,
+                    },
+                )
+                expect(loadedPost.name).to.not.equal("post")
+            }),
+        ))
+})