From 53ce8f60126b4f61708e8e5a73d68d073b91d86c Mon Sep 17 00:00:00 2001 From: Umed Khudoiberdiev Date: Tue, 25 Sep 2018 21:18:21 +0500 Subject: [PATCH 1/5] added sqlitebusy handling logic --- src/driver/sqlite/SqliteConnectionOptions.ts | 13 +++++++- src/driver/sqlite/SqliteQueryRunner.ts | 34 ++++++++++++++------ 2 files changed, 37 insertions(+), 10 deletions(-) diff --git a/src/driver/sqlite/SqliteConnectionOptions.ts b/src/driver/sqlite/SqliteConnectionOptions.ts index 9155047e4f..2abd621053 100644 --- a/src/driver/sqlite/SqliteConnectionOptions.ts +++ b/src/driver/sqlite/SqliteConnectionOptions.ts @@ -20,4 +20,15 @@ export interface SqliteConnectionOptions extends BaseConnectionOptions { */ readonly key?: string; -} \ No newline at end of file + /** + * In your SQLite application when you perform parallel writes its common to face SQLITE_BUSY error. + * This error indicates that SQLite failed to write to the database file since someone else already writes into it. + * Since SQLite cannot handle parallel saves this error cannot be avoided. + * + * To simplify life's of those who have this error this particular option sets a timeout within which ORM will try + * to perform requested write operation again and again until it recieves SQLITE_BUSY error. + * + * Time in milliseconds. + */ + readonly busyErrorRetry?: number; +} diff --git a/src/driver/sqlite/SqliteQueryRunner.ts b/src/driver/sqlite/SqliteQueryRunner.ts index 6e5a55d885..d1ac916d7d 100644 --- a/src/driver/sqlite/SqliteQueryRunner.ts +++ b/src/driver/sqlite/SqliteQueryRunner.ts @@ -1,6 +1,7 @@ import {QueryRunnerAlreadyReleasedError} from "../../error/QueryRunnerAlreadyReleasedError"; import {QueryFailedError} from "../../error/QueryFailedError"; import {AbstractSqliteQueryRunner} from "../sqlite-abstract/AbstractSqliteQueryRunner"; +import {SqliteConnectionOptions} from "./SqliteConnectionOptions"; import {SqliteDriver} from "./SqliteDriver"; import {Broadcaster} from "../../subscriber/Broadcaster"; @@ -36,11 +37,34 @@ export class SqliteQueryRunner extends AbstractSqliteQueryRunner { throw new QueryRunnerAlreadyReleasedError(); const connection = this.driver.connection; + const options = connection.options as SqliteConnectionOptions; return new Promise(async (ok, fail) => { + const databaseConnection = await this.connect(); + this.driver.connection.logger.logQuery(query, parameters, this); + const queryStartTime = +new Date(); + const isInsertQuery = query.substr(0, 11) === "INSERT INTO"; + + const execute = async () => { + if (isInsertQuery) { + databaseConnection.run(query, parameters, handler); + } else { + databaseConnection.all(query, parameters, handler); + } + }; + const handler = function (err: any, result: any) { + if (err) { + if (err.toString().indexOf("SQLITE_BUSY:") !== -1 && + typeof options.busyErrorRetry === "number" && + options.busyErrorRetry) { + setTimeout(execute, options.busyErrorRetry); + return; + } + } + // log slow queries if maxQueryExecution time is set const maxQueryExecutionTime = connection.options.maxQueryExecutionTime; const queryEndTime = +new Date(); @@ -56,15 +80,7 @@ export class SqliteQueryRunner extends AbstractSqliteQueryRunner { } }; - const databaseConnection = await this.connect(); - this.driver.connection.logger.logQuery(query, parameters, this); - const queryStartTime = +new Date(); - const isInsertQuery = query.substr(0, 11) === "INSERT INTO"; - if (isInsertQuery) { - databaseConnection.run(query, parameters, handler); - } else { - databaseConnection.all(query, parameters, handler); - } + await execute(); }); } } \ No newline at end of file From a5c87e417f894afea7921a5a1729e5e760cb95ce Mon Sep 17 00:00:00 2001 From: Umed Khudoiberdiev Date: Tue, 25 Sep 2018 22:47:49 +0500 Subject: [PATCH 2/5] added sqlite wal mode enable logic --- src/driver/sqlite/SqliteConnectionOptions.ts | 11 ++++++++++- src/driver/sqlite/SqliteQueryRunner.ts | 11 ++++++++++- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/src/driver/sqlite/SqliteConnectionOptions.ts b/src/driver/sqlite/SqliteConnectionOptions.ts index 2abd621053..8e092b9c69 100644 --- a/src/driver/sqlite/SqliteConnectionOptions.ts +++ b/src/driver/sqlite/SqliteConnectionOptions.ts @@ -26,9 +26,18 @@ export interface SqliteConnectionOptions extends BaseConnectionOptions { * Since SQLite cannot handle parallel saves this error cannot be avoided. * * To simplify life's of those who have this error this particular option sets a timeout within which ORM will try - * to perform requested write operation again and again until it recieves SQLITE_BUSY error. + * to perform requested write operation again and again until it receives SQLITE_BUSY error. * + * Enabling WAL can improve your app performance and face less SQLITE_BUSY issues. * Time in milliseconds. */ readonly busyErrorRetry?: number; + + /** + * Enables WAL mode. By default its disabled. + * + * @see https://www.sqlite.org/wal.html + */ + readonly enableWAL?: boolean; + } diff --git a/src/driver/sqlite/SqliteQueryRunner.ts b/src/driver/sqlite/SqliteQueryRunner.ts index d1ac916d7d..1658c20186 100644 --- a/src/driver/sqlite/SqliteQueryRunner.ts +++ b/src/driver/sqlite/SqliteQueryRunner.ts @@ -4,6 +4,7 @@ import {AbstractSqliteQueryRunner} from "../sqlite-abstract/AbstractSqliteQueryR import {SqliteConnectionOptions} from "./SqliteConnectionOptions"; import {SqliteDriver} from "./SqliteDriver"; import {Broadcaster} from "../../subscriber/Broadcaster"; +import {IsolationLevel} from "../types/IsolationLevel"; /** * Runs queries on a single sqlite database connection. @@ -83,4 +84,12 @@ export class SqliteQueryRunner extends AbstractSqliteQueryRunner { await execute(); }); } -} \ No newline at end of file + + async startTransaction(isolationLevel?: IsolationLevel): Promise { + if ((this.connection.options as SqliteConnectionOptions).enableWAL === true) { + await this.query("PRAGMA journal_mode = WAL"); + } + + return super.startTransaction(isolationLevel); + } +} From e721f3263cda1d4ebb0358e31c1bb58b838afd98 Mon Sep 17 00:00:00 2001 From: James Ward Date: Fri, 11 Sep 2020 00:30:09 -0400 Subject: [PATCH 3/5] cleaner if block --- src/driver/sqlite/SqliteQueryRunner.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/driver/sqlite/SqliteQueryRunner.ts b/src/driver/sqlite/SqliteQueryRunner.ts index 1658c20186..bc6280595c 100644 --- a/src/driver/sqlite/SqliteQueryRunner.ts +++ b/src/driver/sqlite/SqliteQueryRunner.ts @@ -56,11 +56,8 @@ export class SqliteQueryRunner extends AbstractSqliteQueryRunner { }; const handler = function (err: any, result: any) { - - if (err) { - if (err.toString().indexOf("SQLITE_BUSY:") !== -1 && - typeof options.busyErrorRetry === "number" && - options.busyErrorRetry) { + if (err && err.toString().indexOf("SQLITE_BUSY:") !== -1) { + if (typeof options.busyErrorRetry === "number" && options.busyErrorRetry > 0) { setTimeout(execute, options.busyErrorRetry); return; } From 18011055b4d4a695328d242f44fc0a496b638866 Mon Sep 17 00:00:00 2001 From: James Ward Date: Fri, 11 Sep 2020 00:44:35 -0400 Subject: [PATCH 4/5] move pragma journal mode setting to driver connection --- src/driver/sqlite/SqliteDriver.ts | 4 ++++ src/driver/sqlite/SqliteQueryRunner.ts | 9 --------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/src/driver/sqlite/SqliteDriver.ts b/src/driver/sqlite/SqliteDriver.ts index cdd6226127..db350b13ec 100644 --- a/src/driver/sqlite/SqliteDriver.ts +++ b/src/driver/sqlite/SqliteDriver.ts @@ -105,6 +105,10 @@ export class SqliteDriver extends AbstractSqliteDriver { }); } + if (this.options.enableWAL) { + await run(`PRAGMA journal_mode = WAL;`); + } + // we need to enable foreign keys in sqlite to make sure all foreign key related features // working properly. this also makes onDelete to work with sqlite. await run(`PRAGMA foreign_keys = ON;`); diff --git a/src/driver/sqlite/SqliteQueryRunner.ts b/src/driver/sqlite/SqliteQueryRunner.ts index bc6280595c..308bb72855 100644 --- a/src/driver/sqlite/SqliteQueryRunner.ts +++ b/src/driver/sqlite/SqliteQueryRunner.ts @@ -4,7 +4,6 @@ import {AbstractSqliteQueryRunner} from "../sqlite-abstract/AbstractSqliteQueryR import {SqliteConnectionOptions} from "./SqliteConnectionOptions"; import {SqliteDriver} from "./SqliteDriver"; import {Broadcaster} from "../../subscriber/Broadcaster"; -import {IsolationLevel} from "../types/IsolationLevel"; /** * Runs queries on a single sqlite database connection. @@ -81,12 +80,4 @@ export class SqliteQueryRunner extends AbstractSqliteQueryRunner { await execute(); }); } - - async startTransaction(isolationLevel?: IsolationLevel): Promise { - if ((this.connection.options as SqliteConnectionOptions).enableWAL === true) { - await this.query("PRAGMA journal_mode = WAL"); - } - - return super.startTransaction(isolationLevel); - } } From a7d3efe79da3410bbef8d00d245d2e03207b68e3 Mon Sep 17 00:00:00 2001 From: James Ward Date: Fri, 11 Sep 2020 03:43:02 -0400 Subject: [PATCH 5/5] add enable-wal test --- test/functional/sqlite/enable-wal.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 test/functional/sqlite/enable-wal.ts diff --git a/test/functional/sqlite/enable-wal.ts b/test/functional/sqlite/enable-wal.ts new file mode 100644 index 0000000000..b1af701ebf --- /dev/null +++ b/test/functional/sqlite/enable-wal.ts @@ -0,0 +1,24 @@ +import "reflect-metadata"; +import {expect} from "chai"; +import {Connection} from "../../../src/connection/Connection"; +import {closeTestingConnections, createTestingConnections, reloadTestingDatabases} from "../../utils/test-utils"; + +describe("sqlite driver > enable wal", () => { + let connections: Connection[]; + before(async () => connections = await createTestingConnections({ + entities: [], + enabledDrivers: ["sqlite"], + driverSpecific: { + enableWAL: true + } + })); + beforeEach(() => reloadTestingDatabases(connections)); + after(() => closeTestingConnections(connections)); + + it("should set the journal mode as expected", () => Promise.all(connections.map(async connection => { + // if we come this far, test was successful as a connection was established + const result = await connection.query('PRAGMA journal_mode'); + + expect(result).to.eql([{ journal_mode: 'wal'}]); + }))); +});