feat: implement generated columns for postgres 12 driver

.circleci/config.yml

@@ -128,7 +128,7 @@ jobs:
 
             docker-compose --project-name typeorm --no-ansi up --detach $SERVICES
       - install-packages:
-         cache-key: node<< parameters.node-version >>
+          cache-key: node<< parameters.node-version >>
       - run:
           name: Set up TypeORM Test Runner
           command: |
@@ -213,3 +213,10 @@ workflows:
             - build
           databases: "oracle"
           node-version: "12"
+      - test:
+          name: test (postgres 12) - Node v12
+          requires:
+            - lint
+            - build
+          databases: "postgres-12"
+          node-version: "12"

docker-compose.yml

@@ -50,6 +50,19 @@ services:
       POSTGRES_PASSWORD: "test"
       POSTGRES_DB: "test"
 
+  postgres-12:
+    # mdillon/postgis is postgres + PostGIS (only). if you need additional
+    # extensions, it's probably time to create a purpose-built image with all
+    # necessary extensions. sorry, and thanks for adding support for them!
+    image: "postgis/postgis:12-2.5"
+    container_name: "typeorm-postgres-12"
+    ports:
+      - "5532:5432"
+    environment:
+      POSTGRES_USER: "test"
+      POSTGRES_PASSWORD: "test"
+      POSTGRES_DB: "test"
+
   # mssql
   mssql:
     image: "mcr.microsoft.com/mssql/server:2017-latest-ubuntu"

docs/decorator-reference.md

@@ -194,8 +194,8 @@ If `true`, MySQL automatically adds the `UNSIGNED` attribute to this column.
 * `enum: string[]|AnyEnum` - Used in `enum` column type to specify list of allowed enum values.
 You can specify array of values or specify a enum class.
 * `enumName: string` - A name for generated enum type. If not specified, TypeORM will generate a enum type from entity and column names - so it's neccessary if you intend to use the same enum type in different tables.
-* `asExpression: string` - Generated column expression. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html).
-* `generatedType: "VIRTUAL"|"STORED"` - Generated column type. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html).
+* `asExpression: string` - Generated column expression. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html) and [Postgres](https://www.postgresql.org/docs/12/ddl-generated-columns.html).
+* `generatedType: "VIRTUAL"|"STORED"` - Generated column type. Used only in [MySQL](https://dev.mysql.com/doc/refman/5.7/en/create-table-generated-columns.html)  and [Postgres (Only "STORED")](https://www.postgresql.org/docs/12/ddl-generated-columns.html).
 * `hstoreType: "object"|"string"` - Return type of `HSTORE` column. Returns value as string or as object. Used only in [Postgres](https://www.postgresql.org/docs/9.6/static/hstore.html).
 * `array: boolean` - Used for postgres and cockroachdb column types which can be array (for example int[]).
 * `transformer: ValueTransformer|ValueTransformer[]` - Specifies a value transformer (or array of value transformers) that is to be used to (un)marshal this column when reading or writing to the database. In case of an array, the value transformers will be applied in the natural order from entityValue to databaseValue, and in reverse order from databaseValue to entityValue.

ormconfig.circleci-common.json

@@ -46,6 +46,17 @@
     "database": "test",
     "logging": false
   },
+  {
+    "skip": true,
+    "name": "postgres-12",
+    "type": "postgres",
+    "host": "typeorm-postgres-12",
+    "port": 5432,
+    "username": "test",
+    "password": "test",
+    "database": "test",
+    "logging": false
+  },
   {
     "skip": false,
     "name": "sqljs",

src/driver/postgres/PostgresDriver.ts

@@ -16,6 +16,7 @@ import {ColumnType} from "../types/ColumnTypes";
 import {DataTypeDefaults} from "../types/DataTypeDefaults";
 import {MappedColumnTypes} from "../types/MappedColumnTypes";
 import {ReplicationMode} from "../types/ReplicationMode";
+import {VersionUtils} from "../../util/VersionUtils";
 import {PostgresConnectionCredentialsOptions} from "./PostgresConnectionCredentialsOptions";
 import {PostgresConnectionOptions} from "./PostgresConnectionOptions";
 import {PostgresQueryRunner} from "./PostgresQueryRunner";
@@ -983,7 +984,9 @@ export class PostgresDriver implements Driver {
                 || (tableColumn.enum && columnMetadata.enum && !OrmUtils.isArraysEqual(tableColumn.enum, columnMetadata.enum.map(val => val + ""))) // enums in postgres are always strings
                 || tableColumn.isGenerated !== columnMetadata.isGenerated
                 || (tableColumn.spatialFeatureType || "").toLowerCase() !== (columnMetadata.spatialFeatureType || "").toLowerCase()
-                || tableColumn.srid !== columnMetadata.srid;
+                || tableColumn.srid !== columnMetadata.srid
+                || tableColumn.generatedType !== columnMetadata.generatedType
+                || (tableColumn.asExpression || "").trim() !== (columnMetadata.asExpression || "").trim();
 
             // DEBUG SECTION
             // if (isColumnChanged) {
@@ -1035,6 +1038,15 @@ export class PostgresDriver implements Driver {
         return true;
     }
 
+    /**
+     * Returns true if postgres supports generated columns
+     */
+    async isGeneratedColumnsSupported(runner: QueryRunner): Promise<boolean> {
+        const results = await runner.query("SHOW server_version;", []);
+        const versionString = results[0]["server_version"] as string;
+        return VersionUtils.isGreaterOrEqual(versionString, '12.0');
+    }
+
     /**
      * Returns true if driver supports fulltext indices.
      */

src/driver/postgres/PostgresQueryRunner.ts

@@ -435,6 +435,17 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
             }
         }
 
+        // if table have column with generated type, we must add the expression to the meta table
+        await Promise.all(table.columns
+            .filter(column => column.generatedType === "STORED" && column.asExpression)
+            .map(async column => {
+                const tableName = await this.getTableNameWithSchema(table.name);
+                const deleteQuery = new Query(`DELETE FROM typeorm_generation_meta WHERE table_name = $1 AND column_name = $2`, [tableName, column.name]);
+                upQueries.push(deleteQuery);
+                upQueries.push(new Query(`INSERT INTO typeorm_generation_meta(table_name, column_name, generation_expression) VALUES ($1, $2, $3)`, [tableName, column.name, column.asExpression]));
+                downQueries.push(deleteQuery);
+            }));
+
         upQueries.push(this.createTableSql(table, createForeignKeys));
         downQueries.push(this.dropTableSql(table));
 
@@ -675,6 +686,14 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
             downQueries.push(new Query(`ALTER TABLE ${this.escapePath(table)} DROP CONSTRAINT "${uniqueConstraint.name}"`));
         }
 
+        if (column.generatedType === "STORED" && column.asExpression) {
+            const tableName = await this.getTableNameWithSchema(table.name);
+            const deleteQuery = new Query(`DELETE FROM typeorm_generation_meta WHERE table_name = $1 AND column_name = $2`, [tableName, column.name]);
+            upQueries.push(deleteQuery);
+            upQueries.push(new Query(`INSERT INTO typeorm_generation_meta(table_name, column_name, generation_expression) VALUES ($1, $2, $3)`, [tableName, column.name, column.asExpression]));
+            downQueries.push(deleteQuery);
+        }
+
         // create column's comment
         if (column.comment) {
             upQueries.push(new Query(`COMMENT ON COLUMN ${this.escapePath(table)}."${column.name}" IS ${this.escapeComment(column.comment)}`));
@@ -732,7 +751,12 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
         if (!oldColumn)
             throw new TypeORMError(`Column "${oldTableColumnOrName}" was not found in the "${table.name}" table.`);
 
-        if (oldColumn.type !== newColumn.type || oldColumn.length !== newColumn.length || newColumn.isArray !== oldColumn.isArray) {
+
+        if (oldColumn.type !== newColumn.type
+            || oldColumn.length !== newColumn.length
+            || newColumn.isArray !== oldColumn.isArray
+            || (!oldColumn.generatedType && newColumn.generatedType === "STORED")
+            || (oldColumn.asExpression !== newColumn.asExpression && newColumn.generatedType === "STORED")) {
             // To avoid data conversion, we just recreate column
             await this.dropColumn(table, oldColumn);
             await this.addColumn(table, newColumn);
@@ -1015,6 +1039,24 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
                 downQueries.push(new Query(`ALTER TABLE ${this.escapePath(table)} ALTER COLUMN "${newColumn.name}" TYPE ${this.driver.createFullType(oldColumn)}`));
             }
 
+            if (newColumn.generatedType !== oldColumn.generatedType) {
+                // Convert generated column data to normal column
+                if (!newColumn.generatedType || newColumn.generatedType === "VIRTUAL") {
+                    // We can copy the generated data to the new column
+                    const tableName = await this.getTableNameWithSchema(table.name);
+                    upQueries.push(new Query(`ALTER TABLE ${this.escapePath(table)} RENAME COLUMN "${oldColumn.name}" TO "TEMP_OLD_${oldColumn.name}"`));
+                    upQueries.push(new Query(`ALTER TABLE ${this.escapePath(table)} ADD ${this.buildCreateColumnSql(table, newColumn)}`));
+                    upQueries.push(new Query(`UPDATE ${this.escapePath(table)} SET "${newColumn.name}" = "TEMP_OLD_${oldColumn.name}"`));
+                    upQueries.push(new Query(`ALTER TABLE ${this.escapePath(table)} DROP COLUMN "TEMP_OLD_${oldColumn.name}"`));
+                    upQueries.push(new Query(`DELETE FROM typeorm_generation_meta WHERE table_name = $1 AND column_name = $2`, [tableName, oldColumn.name]));
+                    // However, we can't copy it back on downgrade. It needs to regenerate.
+                    downQueries.push(new Query(`ALTER TABLE ${this.escapePath(table)} DROP COLUMN "${newColumn.name}"`));
+                    downQueries.push(new Query(`ALTER TABLE ${this.escapePath(table)} ADD ${this.buildCreateColumnSql(table, oldColumn)}`));
+                    downQueries.push(new Query(`DELETE FROM typeorm_generation_meta WHERE table_name = $1 AND column_name = $2`, [tableName, newColumn.name]));
+                    downQueries.push(new Query(`INSERT INTO typeorm_generation_meta(table_name, column_name, generation_expression) VALUES ($1, $2, $3)`, [tableName, oldColumn.name, oldColumn.asExpression]));
+                }
+            }
+
         }
 
         await this.executeQueries(upQueries, downQueries);
@@ -1101,6 +1143,12 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
             }
         }
 
+        if (column.generatedType === "STORED") {
+            const tableName = await this.getTableNameWithSchema(table.name);
+            upQueries.push(new Query(`DELETE FROM typeorm_generation_meta WHERE table_name = $1 AND column_name = $2`, [tableName, column.name]));
+            downQueries.push(new Query(`INSERT INTO typeorm_generation_meta(table_name, column_name, generation_expression) VALUES ($1, $2, $3)`, [tableName, column.name, column.asExpression]));
+        }
+
         await this.executeQueries(upQueries, downQueries);
 
         clonedTable.removeColumn(column);
@@ -1818,6 +1866,20 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
                         }
                     }
 
+                    if (dbColumn["is_generated"] === "ALWAYS" && dbColumn["generation_expression"]) {
+                        // In postgres there is no VIRTUAL generated column type
+                        tableColumn.generatedType = "STORED";
+                        // We cannot relay on information_schema.columns.generation_expression, because it is formatted different.
+                        const fullTableName = this.driver.buildTableName(dbTable["table_name"], dbTable["table_schema"]);
+                        const asExpressionQuery = `SELECT * FROM typeorm_generation_meta WHERE table_name = '${await this.getTableNameWithSchema(fullTableName)}' and column_name = '${tableColumn.name}'`;
+                        const results: ObjectLiteral[] = await this.query(asExpressionQuery);
+                        if (results[0] && results[0].generation_expression) {
+                            tableColumn.asExpression = results[0].generation_expression;
+                        } else {
+                            tableColumn.asExpression = "";
+                        }
+                    }
+
                     tableColumn.comment = dbColumn["description"] ? dbColumn["description"] : undefined;
                     if (dbColumn["character_set_name"])
                         tableColumn.charset = dbColumn["character_set_name"];
@@ -2340,6 +2402,21 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
         return `"${tableName}"`;
     }
 
+    /**
+     * Get the table name with table schema
+     * Note: Without ' or "
+     */
+    protected async getTableNameWithSchema(target: Table|string) {
+        const tableName = target instanceof Table ? target.name : target;
+        if (tableName.indexOf(".") === -1) {
+            const schemaResult = await this.query(`SELECT current_schema()`);
+            const schema = schemaResult[0]["current_schema"];
+            return `${schema}.${tableName}`;
+        } else {
+            return `${tableName.split(".")[0]}.${tableName.split(".")[1]}`;
+        }
+    }
+
     /**
      * Builds a query for create column.
      */
@@ -2361,16 +2438,22 @@ export class PostgresQueryRunner extends BaseQueryRunner implements QueryRunner
         } else if (!column.isGenerated || column.type === "uuid") {
             c += " " + this.connection.driver.createFullType(column);
         }
-        if (column.charset)
-            c += " CHARACTER SET \"" + column.charset + "\"";
-        if (column.collation)
-            c += " COLLATE \"" + column.collation + "\"";
-        if (column.isNullable !== true)
-            c += " NOT NULL";
-        if (column.default !== undefined && column.default !== null)
-            c += " DEFAULT " + column.default;
-        if (column.isGenerated && column.generationStrategy === "uuid" && !column.default)
-            c += ` DEFAULT ${this.driver.uuidGenerator}`;
+        // CHARACTER SET, COLLATE, NOT NULL and DEFAULT do not exist on generated (virtual) columns
+        // Also, postgres only supports the stored generated column type
+        if (column.generatedType === "STORED" && column.asExpression) {
+            c += ` GENERATED ALWAYS AS (${column.asExpression}) STORED`;
+        } else {
+            if (column.charset)
+                c += " CHARACTER SET \"" + column.charset + "\"";
+            if (column.collation)
+                c += " COLLATE \"" + column.collation + "\"";
+            if (column.isNullable !== true)
+                c += " NOT NULL";
+            if (column.default !== undefined && column.default !== null)
+                c += " DEFAULT " + column.default;
+            if (column.isGenerated && column.generationStrategy === "uuid" && !column.default)
+                c += ` DEFAULT ${this.driver.uuidGenerator}`;
+        }
 
         return c;
     }

src/schema-builder/RdbmsSchemaBuilder.ts

@@ -19,6 +19,7 @@ import {TableCheck} from "./table/TableCheck";
 import {TableExclusion} from "./table/TableExclusion";
 import {View} from "./view/View";
 import {AuroraDataApiDriver} from "../driver/aurora-data-api/AuroraDataApiDriver";
+import {PostgresConnectionOptions} from "../driver/postgres/PostgresConnectionOptions";
 
 /**
  * Creates complete tables schemas in the database based on the entity metadatas.
@@ -77,8 +78,15 @@ export class RdbmsSchemaBuilder implements SchemaBuilder {
                 await this.createTypeormMetadataTable();
             }
 
+            if (this.connection.driver instanceof PostgresDriver) {
+                if (await this.connection.driver.isGeneratedColumnsSupported(this.queryRunner)) {
+                    await this.createTypeormGeneratedMetadataTable();
+                }
+            }
+
             // Flush the queryrunner table & view cache
             const tablePaths = this.entityToSyncMetadatas.map(metadata => this.getTablePath(metadata));
+
             await this.queryRunner.getTables(tablePaths);
             await this.queryRunner.getViews([]);
 
@@ -825,4 +833,34 @@ export class RdbmsSchemaBuilder implements SchemaBuilder {
         ), true);
     }
 
+    /**
+     * Creates typeorm service table for storing user defined generated columns as expressions.
+     */
+    protected async createTypeormGeneratedMetadataTable() {
+        const options = <PostgresConnectionOptions>this.connection.driver.options;
+        const typeormMetadataTable = this.connection.driver.buildTableName("typeorm_generation_meta", options.schema, options.database);
+
+        await this.queryRunner.createTable(new Table(
+            {
+                name: typeormMetadataTable,
+                columns: [
+                    {
+                        name: "table_name",
+                        type: this.connection.driver.normalizeType({type: this.connection.driver.mappedDataTypes.metadataType}),
+                        isNullable: false,
+                    },
+                    {
+                        name: "column_name",
+                        type: this.connection.driver.normalizeType({type: this.connection.driver.mappedDataTypes.metadataDatabase}),
+                        isNullable: false
+                    },
+                    {
+                        name: "generation_expression",
+                        type: this.connection.driver.normalizeType({type: this.connection.driver.mappedDataTypes.metadataValue}),
+                        isNullable: true
+                    },
+                ]
+            },
+        ), true);
+    }
 }