Return complete list of DDL commands for creating foreign keys in SQL…

…ite (#4194)

lib/dialects/sqlite3/schema/ddl.js

@@ -10,10 +10,11 @@ const invert = require('lodash/invert');
 const isEmpty = require('lodash/isEmpty');
 const negate = require('lodash/negate');
 const omit = require('lodash/omit');
-const uniqueId = require('lodash/uniqueId');
+const { nanonum } = require('../../../util/nanoid');
 const { COMMA_NO_PAREN_REGEX } = require('../../../constants');
 const {
   createTempTable,
+  copyAllData,
   copyData,
   dropTempTable,
   dropOriginal,
@@ -31,7 +32,7 @@ class SQLite3_DDL {
     this.tableCompiler = tableCompiler;
     this.pragma = pragma;
     this.tableNameRaw = this.tableCompiler.tableNameRaw;
-    this.alteredName = uniqueId('_knex_temp_alter');
+    this.alteredName = `_knex_temp_alter${nanonum(3)}`;
     this.connection = connection;
     this.formatter =
       client && client.config && client.config.wrapIdentifier
@@ -77,12 +78,27 @@ class SQLite3_DDL {
     return this.trx.raw(dropTempTable(this.alteredName));
   }
 
-  copyData() {
-    return copyData(this.trx, this.tableName(), this.alteredName);
+  async copyData() {
+    const commands = await copyData(
+      this.trx,
+      this.tableName(),
+      this.alteredName
+    );
+    for (const command of commands) {
+      await this.trx.raw(command);
+    }
   }
 
-  reinsertData(iterator) {
-    return reinsertData(this.trx, iterator, this.tableName(), this.alteredName);
+  async reinsertData(iterator) {
+    const commands = await reinsertData(
+      this.trx,
+      iterator,
+      this.tableName(),
+      this.alteredName
+    );
+    for (const command of commands) {
+      await this.trx.raw(command);
+    }
   }
 
   createTempTable(createTable) {
@@ -479,7 +495,7 @@ class SQLite3_DDL {
           newColumnDefinitions.join(', ')
         );
 
-        return this.reinsertMapped(tableInfo, newSQL, (row) => {
+        return await this.generateReinsertCommands(tableInfo, newSQL, (row) => {
           return row;
         });
       },
@@ -503,6 +519,22 @@ class SQLite3_DDL {
       .then(() => this.reinsertData(mapRow))
       .then(() => this.dropTempTable());
   }
+
+  async generateReinsertCommands(createTable, newSql, mapRow) {
+    const result = [];
+    result.push(
+      createTempTable(createTable, this.tableName(), this.alteredName)
+    );
+
+    result.push(copyAllData(this.tableName(), this.alteredName));
+    result.push(dropOriginal(this.tableName()));
+
+    result.push(newSql);
+
+    result.push(copyAllData(this.alteredName, this.tableName()));
+    result.push(dropTempTable(this.alteredName));
+    return result;
+  }
 }
 
 module.exports = SQLite3_DDL;

lib/dialects/sqlite3/schema/internal/sqlite-ddl-operations.js

@@ -1,21 +1,36 @@
 const identity = require('lodash/identity');
 const chunk = require('lodash/chunk');
 
-async function insertChunked(trx, chunkSize, target, iterator, result) {
+function insertChunked(trx, chunkSize, target, iterator, existingData) {
+  const result = [];
   iterator = iterator || identity;
-  const chunked = chunk(result, chunkSize);
+  const chunked = chunk(existingData, chunkSize);
   for (const batch of chunked) {
-    await trx.queryBuilder().table(target).insert(batch.map(iterator));
+    result.push(
+      trx.queryBuilder().table(target).insert(batch.map(iterator)).toQuery()
+    );
   }
+  return result;
 }
 
 function createTempTable(createTable, tablename, alteredName) {
   return createTable.sql.replace(tablename, alteredName);
 }
 
+// ToDo To be removed
 async function copyData(trx, tableName, alteredName) {
-  const originalData = await trx.raw(`SELECT * FROM "${tableName}"`);
-  return insertChunked(trx, 20, alteredName, identity, originalData);
+  const existingData = await trx.raw(`SELECT * FROM "${tableName}"`);
+  return insertChunked(trx, 20, alteredName, identity, existingData);
+}
+
+// ToDo To be removed
+async function reinsertData(trx, iterator, tableName, alteredName) {
+  const existingData = await trx.raw(`SELECT * FROM "${alteredName}"`);
+  return insertChunked(trx, 20, tableName, iterator, existingData);
+}
+
+function copyAllData(sourceTable, targetTable) {
+  return `INSERT INTO ${targetTable} SELECT * FROM ${sourceTable};`;
 }
 
 function dropOriginal(tableName) {
@@ -26,11 +41,6 @@ function dropTempTable(alteredName) {
   return `DROP TABLE "${alteredName}"`;
 }
 
-async function reinsertData(trx, iterator, tableName, alteredName) {
-  const existingData = await trx.raw(`SELECT * FROM "${alteredName}"`);
-  return insertChunked(trx, 20, tableName, iterator, existingData);
-}
-
 function renameTable(tableName, alteredName) {
   return `ALTER TABLE "${tableName}" RENAME TO "${alteredName}"`;
 }
@@ -40,6 +50,7 @@ function getTableSql(tableName) {
 }
 
 module.exports = {
+  copyAllData,
   copyData,
   createTempTable,
   dropOriginal,

lib/dialects/sqlite3/schema/sqlite-compiler.js

@@ -43,6 +43,30 @@ class SchemaCompiler_SQLite3 extends SchemaCompiler {
       )}`
     );
   }
+
+  async generateDdlCommands() {
+    const sequence = this.builder._sequence;
+    for (let i = 0, l = sequence.length; i < l; i++) {
+      const query = sequence[i];
+      this[query.method].apply(this, query.args);
+    }
+
+    const result = [];
+    const commandSources = this.sequence;
+    for (const commandSource of commandSources) {
+      const command = commandSource.statementsProducer
+        ? await commandSource.statementsProducer()
+        : commandSource.sql;
+
+      if (Array.isArray(command)) {
+        result.push(...command);
+      } else {
+        result.push(command);
+      }
+    }
+
+    return result;
+  }
 }
 
 module.exports = SchemaCompiler_SQLite3;

lib/dialects/sqlite3/schema/sqlite-tablecompiler.js

@@ -157,9 +157,9 @@ class TableCompiler_SQLite3 extends TableCompiler {
 
       this.pushQuery({
         sql: `PRAGMA table_info(${this.tableName()})`,
-        output(pragma) {
+        statementsProducer(pragma, connection) {
           return compiler.client
-            .ddl(compiler, pragma, this.connection)
+            .ddl(compiler, pragma, connection)
             .foreign(foreignInfo);
         },
       });

lib/execution/runner.js

@@ -213,7 +213,16 @@ class Runner {
   // of the queries in sequence.
   async queryArray(queries) {
     if (queries.length === 1) {
-      return this.query(queries[0]);
+      const query = queries[0];
+
+      if (!query.statementsProducer) {
+        return this.query(query);
+      }
+      const statements = await query.statementsProducer(
+        undefined,
+        this.connection
+      );
+      return this.queryArray(statements);
     }
 
     const results = [];

lib/schema/builder.js

@@ -1,4 +1,3 @@
-const { inherits } = require('util');
 const { EventEmitter } = require('events');
 const toArray = require('lodash/toArray');
 const { addQueryContext } = require('../util/helpers');
@@ -11,17 +10,35 @@ const {
 // `knex.builder`, accepting the current `knex` instance,
 // and pulling out the `client` and `grammar` from the current
 // knex instance.
-function SchemaBuilder(client) {
-  this.client = client;
-  this._sequence = [];
+class SchemaBuilder extends EventEmitter {
+  constructor(client) {
+    super();
+    this.client = client;
+    this._sequence = [];
 
-  if (client.config) {
-    this._debug = client.config.debug;
-    saveAsyncStack(this, 4);
+    if (client.config) {
+      this._debug = client.config.debug;
+      saveAsyncStack(this, 4);
+    }
+  }
+
+  withSchema(schemaName) {
+    this._schema = schemaName;
+    return this;
+  }
+
+  toString() {
+    return this.toQuery();
   }
-}
 
-inherits(SchemaBuilder, EventEmitter);
+  toSQL() {
+    return this.client.schemaCompiler(this).toSQL();
+  }
+
+  async generateDdlCommands() {
+    return await this.client.schemaCompiler(this).generateDdlCommands();
+  }
+}
 
 // Each of the schema builder methods just add to the
 // "_sequence" array for consistency.
@@ -69,17 +86,4 @@ inherits(SchemaBuilder, EventEmitter);
 augmentWithBuilderInterface(SchemaBuilder);
 addQueryContext(SchemaBuilder);
 
-SchemaBuilder.prototype.withSchema = function (schemaName) {
-  this._schema = schemaName;
-  return this;
-};
-
-SchemaBuilder.prototype.toString = function () {
-  return this.toQuery();
-};
-
-SchemaBuilder.prototype.toSQL = function () {
-  return this.client.schemaCompiler(this).toSQL();
-};
-
 module.exports = SchemaBuilder;

lib/schema/compiler.js

@@ -60,6 +60,13 @@ class SchemaCompiler {
     }
     return this.sequence;
   }
+
+  async generateDdlCommands() {
+    const generatedCommands = this.toSQL();
+    return Array.isArray(generatedCommands)
+      ? generatedCommands
+      : [generatedCommands];
+  }
 }
 
 SchemaCompiler.prototype.dropTablePrefix = 'drop table ';

lib/util/nanoid.js

@@ -3,6 +3,8 @@
 const urlAlphabet =
   'ModuleSymbhasOwnPr-0123456789ABCDEFGHNRVfgctiUvz_KqYTJkLxpZXIjQW';
 
+const numberAlphabet = '0123456789';
+
 /**
  * Generate URL-friendly unique ID. This method uses the non-secure
  * predictable random generator with bigger collision probability.
@@ -26,4 +28,13 @@ function nanoid(size = 21) {
   return id;
 }
 
-module.exports = { nanoid };
+function nanonum(size = 21) {
+  let id = '';
+  let i = size;
+  while (i--) {
+    id += numberAlphabet[(Math.random() * 10) | 0];
+  }
+  return id;
+}
+
+module.exports = { nanoid, nanonum };