Added option to disable fast import for data generator

Data generator uses CSV imports for a massive speed increase, but can't be used in some environments where SQL admin isn't available. This allows us to set a flag to use the original insert-based importer.
TryGhost · May 9, 2024 · 8c3e5ec · 8c3e5ec
1 parent 56d984f
commit 8c3e5ec
Show file tree

Hide file tree

Showing 3 changed files with 15 additions and 7 deletions.
diff --git a/ghost/data-generator/lib/DataGenerator.js b/ghost/data-generator/lib/DataGenerator.js
@@ -204,10 +204,15 @@ class DataGenerator {
 
     async #run(transaction) {
         if (!DatabaseInfo.isSQLite(this.knex)) {
-            await transaction.raw('ALTER INSTANCE DISABLE INNODB REDO_LOG;');
-            await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
-            await transaction.raw('SET unique_checks=0;');
-            await transaction.raw('SET GLOBAL local_infile=1;');
+            if (process.env.DISABLE_FAST_IMPORT) {
+                await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
+                await transaction.raw('SET unique_checks=0;');
+            } else {
+                await transaction.raw('ALTER INSTANCE DISABLE INNODB REDO_LOG;');
+                await transaction.raw('SET FOREIGN_KEY_CHECKS=0;');
+                await transaction.raw('SET unique_checks=0;');
+                await transaction.raw('SET GLOBAL local_infile=1;');
+            }
         }
 
         if (this.willClearData) {
@@ -274,7 +279,7 @@ class DataGenerator {
         // Re-enable the redo log because it's a persisted global
         // Leaving it disabled can break the database in the event of an unexpected shutdown
         // See https://dev.mysql.com/doc/refman/8.0/en/innodb-redo-log.html#innodb-disable-redo-logging
-        if (!DatabaseInfo.isSQLite(this.knex)) {
+        if (!DatabaseInfo.isSQLite(this.knex) && !process.env.DISABLE_FAST_IMPORT) {
             await transaction.raw('ALTER INSTANCE ENABLE INNODB REDO_LOG;');
         }
     }

diff --git a/ghost/data-generator/lib/importers/TableImporter.js b/ghost/data-generator/lib/importers/TableImporter.js
@@ -101,7 +101,7 @@ class TableImporter {
         const filePath = path.join(rootFolder, `${this.name}.csv`);
         let now = Date.now();
 
-        if (data.length > 5000) {
+        if (data.length > 5000 && !process.env.DISABLE_FAST_IMPORT) {
             try {
                 await fs.promises.unlink(filePath);
             } catch (e) {

diff --git a/ghost/data-generator/lib/utils/JsonImporter.js b/ghost/data-generator/lib/utils/JsonImporter.js
@@ -1,5 +1,8 @@
-class JsonImporter {
+const TableImporter = require('../importers/TableImporter');
+
+class JsonImporter extends TableImporter {
     constructor(knex, transaction) {
+        super();
         this.knex = knex;
         this.transaction = transaction;
     }