feat: implement generated columns for postgres 12 driver (#6469)

* feat: implement generated columns for postgres 12 driver The implementation has the potential to make full text search much faster when using postgres. You can simply pre-generate all tsvector's * test: add tests for generated columns in postgres 12 * docs: document generated columns for postgres 12 * fix: check postgres version for generated columns Generated columns are only available on postgres version 12+ * test: add postgres 12 to tests Currently, there are only tests for postgres 9. This adds postgres 12 as test target * test: remove generated column from model MariaDB will fail with a generated column type * test: use non alpine container for postgres 12 * test: skip generated columns test on mariadb * fix: detect generated column change * fix: circle ci postgres version * fix: add replication mode to isGeneratedColumnsSupported() function Latest changes in master introduce replication mode. This commit adjust the the pull request #6469 to this change * fix: ci testing for postgres 12 Latest changes in master broke the postgres 12 test setup * style: remove SqlServerConnectionOptions generic parameter for createTypeormGeneratedMetadataTable function imnotjames notice this in his review of the pull request * style: remove unnecessary return of Promise.resolve() This return of Promise.resolve() has no effect. We can leave it out * style: fix whitespace issue for config.yml * refactor: use VersionUtils Instead of parsing the version string with parseFloat, use the typeorm VersionUtils * fix: fix failing build After merging the upstream into the pr fork, the build stopped working. The reason why the build fails, is because in the upstream one import is missing and one variable was removed * refactor: replace promise.all() with for loop * refactor: make requested changes * fix: update table name * fix: server version query and escape table column in queries * code refactoring * fixed lint issue * removed "enabledDrivers" from test Co-authored-by: Dmitry Zotov <dmzt08@gmail.com>
typeorm · Nov 14, 2021 · 91080be · 91080be
1 parent c895680
commit 91080be
Show file tree

Hide file tree

Showing 18 changed files with 501 additions and 180 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -128,7 +128,7 @@ jobs:
 
             docker-compose --project-name typeorm --no-ansi up --detach $SERVICES
       - install-packages:
-         cache-key: node<< parameters.node-version >>
+          cache-key: node<< parameters.node-version >>
       - run:
           name: Set up TypeORM Test Runner
           command: |
@@ -213,3 +213,10 @@ workflows:
             - build
           databases: "oracle"
           node-version: "12"
+      - test:
+          name: test (postgres 12) - Node v12
+          requires:
+            - lint
+            - build
+          databases: "postgres-12"
+          node-version: "12"
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -50,6 +50,19 @@ services:
       POSTGRES_PASSWORD: "test"
       POSTGRES_DB: "test"
 
+  postgres-12:
+    # mdillon/postgis is postgres + PostGIS (only). if you need additional
+    # extensions, it's probably time to create a purpose-built image with all
+    # necessary extensions. sorry, and thanks for adding support for them!
+    image: "postgis/postgis:12-2.5"
+    container_name: "typeorm-postgres-12"
+    ports:
+      - "5532:5432"
+    environment:
+      POSTGRES_USER: "test"
+      POSTGRES_PASSWORD: "test"
+      POSTGRES_DB: "test"
+
   # mssql
   mssql:
     image: "mcr.microsoft.com/mssql/server:2017-latest-ubuntu"

diff --git a/docs/decorator-reference.md b/docs/decorator-reference.md
@@ -193,8 +193,8 @@ If `true`, MySQL automatically adds the `UNSIGNED` attribute to this column.
 * `enum: string[]|AnyEnum` - Used in `enum` column type to specify list of allowed enum values.
 You can specify array of values or specify a enum class.
 * `enumName: string` - A name for generated enum type. If not specified, TypeORM will generate a enum type from entity and column names - so it's neccessary if you intend to use the same enum type in different tables.
-* `asExpression: string` - Generated column expression. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html).
-* `generatedType: "VIRTUAL"|"STORED"` - Generated column type. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html).
+* `asExpression: string` - Generated column expression. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html) and [Postgres](https://www.postgresql.org/docs/12/ddl-generated-columns.html).
+* `generatedType: "VIRTUAL"|"STORED"` - Generated column type. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html)  and [Postgres (Only "STORED")](https://www.postgresql.org/docs/12/ddl-generated-columns.html).
 * `hstoreType: "object"|"string"` - Return type of `HSTORE` column. Returns value as string or as object. Used only in [Postgres](https://www.postgresql.org/docs/9.6/static/hstore.html).
 * `array: boolean` - Used for postgres and cockroachdb column types which can be array (for example int[]).
 * `transformer: ValueTransformer|ValueTransformer[]` - Specifies a value transformer (or array of value transformers) that is to be used to (un)marshal this column when reading or writing to the database. In case of an array, the value transformers will be applied in the natural order from entityValue to databaseValue, and in reverse order from databaseValue to entityValue.

diff --git a/ormconfig.circleci-common.json b/ormconfig.circleci-common.json
@@ -46,6 +46,17 @@
     "database": "test",
     "logging": false
   },
+  {
+    "skip": true,
+    "name": "postgres-12",
+    "type": "postgres",
+    "host": "typeorm-postgres-12",
+    "port": 5432,
+    "username": "test",
+    "password": "test",
+    "database": "test",
+    "logging": false
+  },
   {
     "skip": false,
     "name": "sqljs",

diff --git a/src/driver/aurora-data-api/AuroraDataApiQueryRunner.ts b/src/driver/aurora-data-api/AuroraDataApiQueryRunner.ts
@@ -22,6 +22,7 @@ import {TableCheck} from "../../schema-builder/table/TableCheck";
 import {IsolationLevel} from "../types/IsolationLevel";
 import {TableExclusion} from "../../schema-builder/table/TableExclusion";
 import { TypeORMError } from "../../error";
+import {MetadataTableType} from "../types/MetadataTableType";
 
 /**
  * Runs queries on a single mysql database connection.
@@ -1172,7 +1173,7 @@ export class AuroraDataApiQueryRunner extends BaseQueryRunner implements QueryRu
         }).join(" OR ");
 
         const query = `SELECT \`t\`.*, \`v\`.\`check_option\` FROM ${this.escapePath(this.getTypeormMetadataTableName())} \`t\` ` +
-            `INNER JOIN \`information_schema\`.\`views\` \`v\` ON \`v\`.\`table_schema\` = \`t\`.\`schema\` AND \`v\`.\`table_name\` = \`t\`.\`name\` WHERE \`t\`.\`type\` = 'VIEW' ${viewsCondition ? `AND (${viewsCondition})` : ""}`;
+            `INNER JOIN \`information_schema\`.\`views\` \`v\` ON \`v\`.\`table_schema\` = \`t\`.\`schema\` AND \`v\`.\`table_name\` = \`t\`.\`name\` WHERE \`t\`.\`type\` = '${MetadataTableType.VIEW}' ${viewsCondition ? `AND (${viewsCondition})` : ""}`;
         const dbViews = await this.query(query);
         return dbViews.map((dbView: any) => {
             const view = new View();
@@ -1532,13 +1533,12 @@ export class AuroraDataApiQueryRunner extends BaseQueryRunner implements QueryRu
     protected async insertViewDefinitionSql(view: View): Promise<Query> {
         const currentDatabase = await this.getCurrentDatabase();
         const expression = typeof view.expression === "string" ? view.expression.trim() : view.expression(this.connection).getQuery();
-        const [query, parameters] = this.connection.createQueryBuilder()
-            .insert()
-            .into(this.getTypeormMetadataTableName())
-            .values({ type: "VIEW", schema: currentDatabase, name: view.name, value: expression })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        return this.insertTypeormMetadataSql({
+            type: MetadataTableType.VIEW,
+            schema: currentDatabase,
+            name: view.name,
+            value: expression
+        });
     }
 
     /**
@@ -1554,15 +1554,7 @@ export class AuroraDataApiQueryRunner extends BaseQueryRunner implements QueryRu
     protected async deleteViewDefinitionSql(viewOrPath: View|string): Promise<Query> {
         const currentDatabase = await this.getCurrentDatabase();
         const viewName = viewOrPath instanceof View ? viewOrPath.name : viewOrPath;
-        const qb = this.connection.createQueryBuilder();
-        const [query, parameters] = qb.delete()
-            .from(this.getTypeormMetadataTableName())
-            .where(`${qb.escape("type")} = 'VIEW'`)
-            .andWhere(`${qb.escape("schema")} = :schema`, { schema: currentDatabase })
-            .andWhere(`${qb.escape("name")} = :name`, { name: viewName })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        return this.deleteTypeormMetadataSql({ type: MetadataTableType.VIEW, schema: currentDatabase, name: viewName });
     }
 
     /**

diff --git a/src/driver/cockroachdb/CockroachQueryRunner.ts b/src/driver/cockroachdb/CockroachQueryRunner.ts
@@ -24,6 +24,7 @@ import {IsolationLevel} from "../types/IsolationLevel";
 import {TableExclusion} from "../../schema-builder/table/TableExclusion";
 import {ReplicationMode} from "../types/ReplicationMode";
 import { TypeORMError } from "../../error";
+import {MetadataTableType} from "../types/MetadataTableType";
 
 /**
  * Runs queries on a single postgres database connection.
@@ -1380,7 +1381,7 @@ export class CockroachQueryRunner extends BaseQueryRunner implements QueryRunner
         }).join(" OR ");
 
         const query = `SELECT "t".*, "v"."check_option" FROM ${this.escapePath(this.getTypeormMetadataTableName())} "t" ` +
-            `INNER JOIN "information_schema"."views" "v" ON "v"."table_schema" = "t"."schema" AND "v"."table_name" = "t"."name" WHERE "t"."type" = 'VIEW' ${viewsCondition ? `AND (${viewsCondition})` : ""}`;
+            `INNER JOIN "information_schema"."views" "v" ON "v"."table_schema" = "t"."schema" AND "v"."table_name" = "t"."name" WHERE "t"."type" = '${MetadataTableType.VIEW}' ${viewsCondition ? `AND (${viewsCondition})` : ""}`;
         const dbViews = await this.query(query);
         return dbViews.map((dbView: any) => {
             const view = new View();
@@ -1798,13 +1799,12 @@ export class CockroachQueryRunner extends BaseQueryRunner implements QueryRunner
         }
 
         const expression = typeof view.expression === "string" ? view.expression.trim() : view.expression(this.connection).getQuery();
-        const [query, parameters] = this.connection.createQueryBuilder()
-            .insert()
-            .into(this.getTypeormMetadataTableName())
-            .values({ type: "VIEW", schema: schema, name: name, value: expression })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        return this.insertTypeormMetadataSql({
+            type: MetadataTableType.VIEW,
+            schema: schema,
+            name: name,
+            value: expression
+        });
     }
 
     /**
@@ -1826,15 +1826,7 @@ export class CockroachQueryRunner extends BaseQueryRunner implements QueryRunner
             schema = currentSchema;
         }
 
-        const qb = this.connection.createQueryBuilder();
-        const [query, parameters] = qb.delete()
-            .from(this.getTypeormMetadataTableName())
-            .where(`${qb.escape("type")} = 'VIEW'`)
-            .andWhere(`${qb.escape("schema")} = :schema`, { schema })
-            .andWhere(`${qb.escape("name")} = :name`, { name })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        return this.deleteTypeormMetadataSql({ type: MetadataTableType.VIEW, schema, name });
     }
 
     /**

diff --git a/src/driver/mysql/MysqlQueryRunner.ts b/src/driver/mysql/MysqlQueryRunner.ts
@@ -25,6 +25,7 @@ import {TableExclusion} from "../../schema-builder/table/TableExclusion";
 import {VersionUtils} from "../../util/VersionUtils";
 import {ReplicationMode} from "../types/ReplicationMode";
 import { TypeORMError } from "../../error";
+import {MetadataTableType} from "../types/MetadataTableType";
 
 /**
  * Runs queries on a single mysql database connection.
@@ -1227,7 +1228,7 @@ export class MysqlQueryRunner extends BaseQueryRunner implements QueryRunner {
         }).join(" OR ");
 
         const query = `SELECT \`t\`.*, \`v\`.\`check_option\` FROM ${this.escapePath(this.getTypeormMetadataTableName())} \`t\` ` +
-            `INNER JOIN \`information_schema\`.\`views\` \`v\` ON \`v\`.\`table_schema\` = \`t\`.\`schema\` AND \`v\`.\`table_name\` = \`t\`.\`name\` WHERE \`t\`.\`type\` = 'VIEW' ${viewsCondition ? `AND (${viewsCondition})` : ""}`;
+            `INNER JOIN \`information_schema\`.\`views\` \`v\` ON \`v\`.\`table_schema\` = \`t\`.\`schema\` AND \`v\`.\`table_name\` = \`t\`.\`name\` WHERE \`t\`.\`type\` = '${MetadataTableType.VIEW}' ${viewsCondition ? `AND (${viewsCondition})` : ""}`;
         const dbViews = await this.query(query);
         return dbViews.map((dbView: any) => {
             const view = new View();
@@ -1720,13 +1721,12 @@ export class MysqlQueryRunner extends BaseQueryRunner implements QueryRunner {
     protected async insertViewDefinitionSql(view: View): Promise<Query> {
         const currentDatabase = await this.getCurrentDatabase();
         const expression = typeof view.expression === "string" ? view.expression.trim() : view.expression(this.connection).getQuery();
-        const [query, parameters] = this.connection.createQueryBuilder()
-            .insert()
-            .into(this.getTypeormMetadataTableName())
-            .values({ type: "VIEW", schema: currentDatabase, name: view.name, value: expression })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        return this.insertTypeormMetadataSql({
+            type: MetadataTableType.VIEW,
+            schema: currentDatabase,
+            name: view.name,
+            value: expression
+        });
     }
 
     /**
@@ -1742,15 +1742,11 @@ export class MysqlQueryRunner extends BaseQueryRunner implements QueryRunner {
     protected async deleteViewDefinitionSql(viewOrPath: View|string): Promise<Query> {
         const currentDatabase = await this.getCurrentDatabase();
         const viewName = viewOrPath instanceof View ? viewOrPath.name : viewOrPath;
-        const qb = this.connection.createQueryBuilder();
-        const [query, parameters] = qb.delete()
-            .from(this.getTypeormMetadataTableName())
-            .where(`${qb.escape("type")} = 'VIEW'`)
-            .andWhere(`${qb.escape("schema")} = :schema`, { schema: currentDatabase })
-            .andWhere(`${qb.escape("name")} = :name`, { name: viewName })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        return this.deleteTypeormMetadataSql({
+            type: MetadataTableType.VIEW,
+            schema: currentDatabase,
+            name: viewName
+        });
     }
 
     /**

diff --git a/src/driver/oracle/OracleQueryRunner.ts b/src/driver/oracle/OracleQueryRunner.ts
@@ -23,6 +23,7 @@ import {TableExclusion} from "../../schema-builder/table/TableExclusion";
 import {ReplicationMode} from "../types/ReplicationMode";
 import { TypeORMError } from "../../error";
 import { QueryResult } from "../../query-runner/QueryResult";
+import {MetadataTableType} from "../types/MetadataTableType";
 
 /**
  * Runs queries on a single oracle database connection.
@@ -1248,7 +1249,7 @@ export class OracleQueryRunner extends BaseQueryRunner implements QueryRunner {
         const viewNamesString = viewNames.map(name => "'" + name + "'").join(", ");
         let query = `SELECT "T".* FROM ${this.escapePath(this.getTypeormMetadataTableName())} "T" ` +
             `INNER JOIN "USER_OBJECTS" "O" ON "O"."OBJECT_NAME" = "T"."name" AND "O"."OBJECT_TYPE" IN ( 'MATERIALIZED VIEW', 'VIEW' ) ` +
-            `WHERE "T"."type" IN ( 'MATERIALIZED_VIEW', 'VIEW' )`;
+            `WHERE "T"."type" IN ( '${MetadataTableType.MATERIALIZED_VIEW}', '${MetadataTableType.VIEW}' )`;
         if (viewNamesString.length > 0)
             query += ` AND "T"."name" IN (${viewNamesString})`;
         const dbViews = await this.query(query);
@@ -1260,7 +1261,7 @@ export class OracleQueryRunner extends BaseQueryRunner implements QueryRunner {
             view.schema = parsedName.schema || dbView["schema"] || currentSchema;
             view.name = parsedName.tableName;
             view.expression = dbView["value"];
-            view.materialized = dbView["type"] === "MATERIALIZED_VIEW";
+            view.materialized = dbView["type"] === MetadataTableType.MATERIALIZED_VIEW;
             return view;
         });
     }
@@ -1584,14 +1585,8 @@ export class OracleQueryRunner extends BaseQueryRunner implements QueryRunner {
 
     protected insertViewDefinitionSql(view: View): Query {
         const expression = typeof view.expression === "string" ? view.expression.trim() : view.expression(this.connection).getQuery();
-        const type = view.materialized ? "MATERIALIZED_VIEW" : "VIEW"
-        const [query, parameters] = this.connection.createQueryBuilder()
-            .insert()
-            .into(this.getTypeormMetadataTableName())
-            .values({ type: type, name: view.name, value: expression })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        const type = view.materialized ? MetadataTableType.MATERIALIZED_VIEW : MetadataTableType.VIEW;
+        return this.insertTypeormMetadataSql({ type: type, name: view.name, value: expression });
     }
 
     /**
@@ -1606,15 +1601,8 @@ export class OracleQueryRunner extends BaseQueryRunner implements QueryRunner {
      * Builds remove view sql.
      */
     protected deleteViewDefinitionSql(view: View): Query {
-        const qb = this.connection.createQueryBuilder();
-        const type = view.materialized ? "MATERIALIZED_VIEW" : "VIEW"
-        const [query, parameters] = qb.delete()
-            .from(this.getTypeormMetadataTableName())
-            .where(`${qb.escape("type")} = :type`, { type })
-            .andWhere(`${qb.escape("name")} = :name`, { name: view.name })
-            .getQueryAndParameters();
-
-        return new Query(query, parameters);
+        const type = view.materialized ? MetadataTableType.MATERIALIZED_VIEW : MetadataTableType.VIEW;
+        return this.deleteTypeormMetadataSql({ type, name: view.name });
     }
 
     /**

diff --git a/src/driver/postgres/PostgresDriver.ts b/src/driver/postgres/PostgresDriver.ts
@@ -16,6 +16,7 @@ import {ColumnType} from "../types/ColumnTypes";
 import {DataTypeDefaults} from "../types/DataTypeDefaults";
 import {MappedColumnTypes} from "../types/MappedColumnTypes";
 import {ReplicationMode} from "../types/ReplicationMode";
+import {VersionUtils} from "../../util/VersionUtils";
 import {PostgresConnectionCredentialsOptions} from "./PostgresConnectionCredentialsOptions";
 import {PostgresConnectionOptions} from "./PostgresConnectionOptions";
 import {PostgresQueryRunner} from "./PostgresQueryRunner";
@@ -270,6 +271,8 @@ export class PostgresDriver implements Driver {
      */
     maxAliasLength = 63;
 
+    isGeneratedColumnsSupported: boolean = false;
+
     // -------------------------------------------------------------------------
     // Constructor
     // -------------------------------------------------------------------------
@@ -346,13 +349,22 @@ export class PostgresDriver implements Driver {
      */
     async afterConnect(): Promise<void> {
         const extensionsMetadata = await this.checkMetadataForExtensions();
+        const [ connection, release ] = await this.obtainMasterConnection()
 
         const installExtensions = this.options.installExtensions === undefined || this.options.installExtensions;
         if (installExtensions && extensionsMetadata.hasExtensions) {
-            const [ connection, release ] = await this.obtainMasterConnection()
             await this.enableExtensions(extensionsMetadata, connection);
-            await release()
         }
+
+        const results = await this.executeQuery(connection, "SHOW server_version;") as {
+            rows: {
+                server_version: string;
+            }[];
+        };
+        const versionString = results.rows[0].server_version;
+        this.isGeneratedColumnsSupported = VersionUtils.isGreaterOrEqual(versionString, "12.0");
+
+        await release()
     }
 
     protected async enableExtensions(extensionsMetadata: any, connection: any) {
@@ -1010,7 +1022,9 @@ export class PostgresDriver implements Driver {
                 || (tableColumn.enum && columnMetadata.enum && !OrmUtils.isArraysEqual(tableColumn.enum, columnMetadata.enum.map(val => val + ""))) // enums in postgres are always strings
                 || tableColumn.isGenerated !== columnMetadata.isGenerated
                 || (tableColumn.spatialFeatureType || "").toLowerCase() !== (columnMetadata.spatialFeatureType || "").toLowerCase()
-                || tableColumn.srid !== columnMetadata.srid;
+                || tableColumn.srid !== columnMetadata.srid
+                || tableColumn.generatedType !== columnMetadata.generatedType
+                || (tableColumn.asExpression || "").trim() !== (columnMetadata.asExpression || "").trim();
 
             // DEBUG SECTION
             // if (isColumnChanged) {