Skip to content

Commit

Permalink
feat: sqlite attach (#8396)
Browse files Browse the repository at this point in the history 
* Updated to latest master. Cleans up codestyle, ignores yarn

* re-adds the portable path tests

* mapping bug fix that appears with unrecognised existing tables

* Review comments

* review comments

* test breakage with node 12
  • Loading branch information
@elmpp
elmpp committed Feb 16, 2022
1 parent 31f0b55 commit 9e844d9
Show file tree
Hide file tree
Showing 22 changed files with 10,658 additions and 66 deletions.
4 changes: 4 additions & 0 deletions .gitignore
View file
Expand Up @@ -9,3 +9,7 @@ node_modules/
ormlogs.log
npm-debug.log
/test/github-issues/799/tmp/*
# ignore yarn2 artifacts but allow yarn.lock (forces yarn1 compat which is node_modules)
.yarn/
.yarn*

26 changes: 12 additions & 14 deletions package.json
View file
Expand Up @@ -21,25 +21,25 @@
"types": "./index.d.ts",
"type": "commonjs",
"browser": {
"./browser/connection/ConnectionOptionsReader.js": "./browser/platform/BrowserConnectionOptionsReaderDummy.js",
"./browser/connection/options-reader/ConnectionOptionsXmlReader.js": "./browser/platform/BrowserConnectionOptionsReaderDummy.js",
"./browser/connection/options-reader/ConnectionOptionsYmlReader.js": "./browser/platform/BrowserConnectionOptionsReaderDummy.js",
"./browser/driver/aurora-data-api/AuroraDataApiDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/better-sqlite3/BetterSqlite3Driver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/cockroachdb/CockroachDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/postgres/PostgresDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/mongodb/MongoDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/mongodb/MongoQueryRunner.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/mysql/MysqlDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/oracle/OracleDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/postgres/PostgresDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/sap/SapDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/mysql/MysqlDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/sqlite/SqliteDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/sqlserver/SqlServerDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/mongodb/MongoDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/mongodb/MongoQueryRunner.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/entity-manager/MongoEntityManager.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/repository/MongoRepository.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/sqlite/SqliteDriver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/driver/better-sqlite3/BetterSqlite3Driver.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/util/DirectoryExportedClassesLoader.js": "./browser/platform/BrowserDirectoryExportedClassesLoader.js",
"./browser/logger/FileLogger.js": "./browser/platform/BrowserFileLoggerDummy.js",
"./browser/connection/ConnectionOptionsReader.js": "./browser/platform/BrowserConnectionOptionsReaderDummy.js",
"./browser/connection/options-reader/ConnectionOptionsXmlReader.js": "./browser/platform/BrowserConnectionOptionsReaderDummy.js",
"./browser/connection/options-reader/ConnectionOptionsYmlReader.js": "./browser/platform/BrowserConnectionOptionsReaderDummy.js",
"./browser/platform/PlatformTools.js": "./browser/platform/BrowserPlatformTools.js",
"./browser/repository/MongoRepository.js": "./browser/platform/BrowserDisabledDriversDummy.js",
"./browser/util/DirectoryExportedClassesLoader.js": "./browser/platform/BrowserDirectoryExportedClassesLoader.js",
"./index.js": "./browser/index.js"
},
"repository": {
Expand Down Expand Up @@ -223,9 +223,7 @@
"lint": "eslint -c ./.eslintrc.js src/**/*.ts test/**/*.ts sample/**/*.ts",
"changelog": "conventional-changelog -p angular -i CHANGELOG.md -s -r 2"
},
"bin": {
"typeorm": "./cli.js"
},
"bin": "./cli.js",
"funding": "https://opencollective.com/typeorm",
"collective": {
"type": "opencollective",
Expand Down
6 changes: 6 additions & 0 deletions src/connection/BaseConnectionOptions.ts
View file
Expand Up @@ -121,6 +121,12 @@ export interface BaseConnectionOptions {
*/
readonly extra?: any;

/**
* Holds reference to the baseDirectory where configuration file are expected
* @internal
*/
baseDirectory?: string;

/**
* Allows to setup cache options.
*/
Expand Down
17 changes: 13 additions & 4 deletions src/connection/Connection.ts
View file
Expand Up @@ -33,6 +33,7 @@ import {SqljsEntityManager} from "../entity-manager/SqljsEntityManager";
import {RelationLoader} from "../query-builder/RelationLoader";
import {EntitySchema} from "../entity-schema/EntitySchema";
import {SqlServerDriver} from "../driver/sqlserver/SqlServerDriver";
import {AbstractSqliteDriver} from "../driver/sqlite-abstract/AbstractSqliteDriver";
import {MysqlDriver} from "../driver/mysql/MysqlDriver";
import {ObjectUtils} from "../util/ObjectUtils";
import {IsolationLevel} from "../driver/types/IsolationLevel";
Expand Down Expand Up @@ -260,15 +261,23 @@ export class Connection {
async dropDatabase(): Promise<void> {
const queryRunner = this.createQueryRunner();
try {
if (this.driver instanceof SqlServerDriver || this.driver instanceof MysqlDriver || this.driver instanceof AuroraDataApiDriver) {
const databases: string[] = this.driver.database ? [this.driver.database] : [];
if (this.driver instanceof SqlServerDriver || this.driver instanceof MysqlDriver || this.driver instanceof AuroraDataApiDriver || this.driver instanceof AbstractSqliteDriver) {
const databases: string[] = [];
this.entityMetadatas.forEach(metadata => {
if (metadata.database && databases.indexOf(metadata.database) === -1)
databases.push(metadata.database);
});
if (databases.length === 0 && this.driver.database) {
databases.push(this.driver.database);
};

for (const database of databases) {
await queryRunner.clearDatabase(database);
if (databases.length === 0) {
await queryRunner.clearDatabase();
}
else {
for (const database of databases) {
await queryRunner.clearDatabase(database);
}
}
} else {
await queryRunner.clearDatabase();
Expand Down
4 changes: 3 additions & 1 deletion src/connection/ConnectionOptionsReader.ts
View file
Expand Up @@ -6,6 +6,7 @@ import {ConnectionOptionsEnvReader} from "./options-reader/ConnectionOptionsEnvR
import {ConnectionOptionsYmlReader} from "./options-reader/ConnectionOptionsYmlReader";
import {ConnectionOptionsXmlReader} from "./options-reader/ConnectionOptionsXmlReader";
import { TypeORMError } from "../error";
import { isAbsolute } from "../util/PathUtils";
import {importOrRequireFile} from "../util/ImportUtils";

/**
Expand Down Expand Up @@ -151,6 +152,7 @@ export class ConnectionOptionsReader {
connectionOptions = [connectionOptions];

connectionOptions.forEach(options => {
options.baseDirectory = this.baseDirectory;
if (options.entities) {
const entities = (options.entities as any[]).map(entity => {
if (typeof entity === "string" && entity.substr(0, 1) !== "/")
Expand Down Expand Up @@ -181,7 +183,7 @@ export class ConnectionOptionsReader {

// make database path file in sqlite relative to package.json
if (options.type === "sqlite" || options.type === "better-sqlite3") {
if (typeof options.database === "string" &&
if (typeof options.database === "string" && !isAbsolute(options.database) &&
options.database.substr(0, 1) !== "/" && // unix absolute
options.database.substr(1, 2) !== ":\\" && // windows absolute
options.database !== ":memory:") {
Expand Down
55 changes: 52 additions & 3 deletions src/driver/better-sqlite3/BetterSqlite3Driver.ts
View file
Expand Up @@ -10,6 +10,7 @@ import { AbstractSqliteDriver } from "../sqlite-abstract/AbstractSqliteDriver";
import { BetterSqlite3ConnectionOptions } from "./BetterSqlite3ConnectionOptions";
import { BetterSqlite3QueryRunner } from "./BetterSqlite3QueryRunner";
import {ReplicationMode} from "../types/ReplicationMode";
import { filepathToName, isAbsolute } from "../../util/PathUtils";

/**
* Organizes communication with sqlite DBMS.
Expand Down Expand Up @@ -79,6 +80,34 @@ export class BetterSqlite3Driver extends AbstractSqliteDriver {
return super.normalizeType(column);
}

async afterConnect(): Promise<void> {
return this.attachDatabases();
}

/**
* For SQLite, the database may be added in the decorator metadata. It will be a filepath to a database file.
*/
buildTableName(tableName: string, _schema?: string, database?: string): string {

if (!database) return tableName;
if (this.getAttachedDatabaseHandleByRelativePath(database)) return `${this.getAttachedDatabaseHandleByRelativePath(database)}.${tableName}`;

if (database === this.options.database) return tableName;

// we use the decorated name as supplied when deriving attach handle (ideally without non-portable absolute path)
const identifierHash = filepathToName(database);
// decorated name will be assumed relative to main database file when non absolute. Paths supplied as absolute won't be portable
const absFilepath = isAbsolute(database) ? database : path.join(this.getMainDatabasePath(), database);

this.attachedDatabases[database] = {
attachFilepathAbsolute: absFilepath,
attachFilepathRelative: database,
attachHandle: identifierHash,
};

return `${identifierHash}.${tableName}`;
}

// -------------------------------------------------------------------------
// Protected Methods
// -------------------------------------------------------------------------
Expand All @@ -89,7 +118,7 @@ export class BetterSqlite3Driver extends AbstractSqliteDriver {
protected async createDatabaseConnection() {
// not to create database directory if is in memory
if (this.options.database !== ":memory:")
await this.createDatabaseDirectory(this.options.database);
await this.createDatabaseDirectory(path.dirname(this.options.database));

const {
database,
Expand Down Expand Up @@ -137,8 +166,28 @@ export class BetterSqlite3Driver extends AbstractSqliteDriver {
/**
* Auto creates database directory if it does not exist.
*/
protected async createDatabaseDirectory(fullPath: string): Promise<void> {
await mkdirp(path.dirname(fullPath));
protected async createDatabaseDirectory(dbPath: string): Promise<void> {
await mkdirp(dbPath);
}

/**
* Performs the attaching of the database files. The attachedDatabase should have been populated during calls to #buildTableName
* during EntityMetadata production (see EntityMetadata#buildTablePath)
*
* https://sqlite.org/lang_attach.html
*/
protected async attachDatabases() {

// @todo - possibly check number of databases (but unqueriable at runtime sadly) - https://www.sqlite.org/limits.html#max_attached
for await (const {attachHandle, attachFilepathAbsolute} of Object.values(this.attachedDatabases)) {
await this.createDatabaseDirectory(path.dirname(attachFilepathAbsolute));
await this.connection.query(`ATTACH "${attachFilepathAbsolute}" AS "${attachHandle}"`);
}
}

protected getMainDatabasePath(): string {
const optionsDb = this.options.database;
return path.dirname(isAbsolute(optionsDb) ? optionsDb : path.join(this.options.baseDirectory!, optionsDb));
}

}
15 changes: 15 additions & 0 deletions src/driver/better-sqlite3/BetterSqlite3QueryRunner.ts
View file
Expand Up @@ -128,4 +128,19 @@ export class BetterSqlite3QueryRunner extends AbstractSqliteQueryRunner {
throw new QueryFailedError(query, parameters, err);
}
}

// -------------------------------------------------------------------------
// Protected Methods
// -------------------------------------------------------------------------

protected async loadTableRecords(tablePath: string, tableOrIndex: "table" | "index") {
const [database, tableName] = this.splitTablePath(tablePath);
const res = await this.query(`SELECT ${database ? `'${database}'` : null} as database, * FROM ${this.escapePath(`${database ? `${database}.` : ""}sqlite_master`)} WHERE "type" = '${tableOrIndex}' AND "${tableOrIndex === "table" ? "name" : "tbl_name"}" IN ('${tableName}')`);
return res;
}
protected async loadPragmaRecords(tablePath: string, pragma: string) {
const [database, tableName] = this.splitTablePath(tablePath);
const res = await this.query(`PRAGMA ${database ? `"${database}".` : ""}${pragma}("${tableName}")`);
return res;
}
}
33 changes: 31 additions & 2 deletions src/driver/sqlite-abstract/AbstractSqliteDriver.ts
View file
Expand Up @@ -20,6 +20,13 @@ import { Table } from "../../schema-builder/table/Table";
import { View } from "../../schema-builder/view/View";
import { TableForeignKey } from "../../schema-builder/table/TableForeignKey";


type DatabasesMap = Record<string, {
attachFilepathAbsolute: string
attachFilepathRelative: string
attachHandle: string
}>;

/**
* Organizes communication with sqlite DBMS.
*/
Expand Down Expand Up @@ -209,6 +216,15 @@ export abstract class AbstractSqliteDriver implements Driver {
*/
maxAliasLength?: number;

// -------------------------------------------------------------------------
// Protected Properties
// -------------------------------------------------------------------------

/**
* Any attached databases (excepting default 'main')
*/
attachedDatabases: DatabasesMap = {};

// -------------------------------------------------------------------------
// Constructor
// -------------------------------------------------------------------------
Expand Down Expand Up @@ -257,6 +273,18 @@ export abstract class AbstractSqliteDriver implements Driver {
});
}

hasAttachedDatabases(): boolean {
return !!Object.keys(this.attachedDatabases).length;
}

getAttachedDatabaseHandleByRelativePath(path: string): string | undefined {
return this.attachedDatabases?.[path]?.attachHandle
}

getAttachedDatabasePathRelativeByHandle(handle: string): string | undefined {
return Object.values(this.attachedDatabases).find(({attachHandle}) => handle === attachHandle)?.attachFilepathRelative
}

/**
* Creates a schema builder used to build and sync a schema.
*/
Expand Down Expand Up @@ -429,7 +457,7 @@ export abstract class AbstractSqliteDriver implements Driver {
const driverSchema = undefined

if (target instanceof Table || target instanceof View) {
const parsed = this.parseTableName(target.name);
const parsed = this.parseTableName(target.schema ? `"${target.schema}"."${target.name}"` : target.name);

return {
database: target.database || parsed.database || driverDatabase,
Expand Down Expand Up @@ -468,8 +496,9 @@ export abstract class AbstractSqliteDriver implements Driver {
tableName: parts[2]
};
} else if (parts.length === 2) {
const database = this.getAttachedDatabasePathRelativeByHandle(parts[0]) ?? driverDatabase
return {
database: driverDatabase,
database: database,
schema: parts[0],
tableName: parts[1]
};
Expand Down

0 comments on commit 9e844d9

Please sign in to comment.