Implement support for custom seed sources (#4842)

Co-authored-by: maximelkin <maxelkin@list.ru>

lib/migrations/common/MigrationsLoader.js

@@ -0,0 +1,36 @@
+const path = require('path');
+const DEFAULT_LOAD_EXTENSIONS = Object.freeze([
+  '.co',
+  '.coffee',
+  '.eg',
+  '.iced',
+  '.js',
+  '.cjs',
+  '.litcoffee',
+  '.ls',
+  '.ts',
+]);
+
+class AbstractMigrationsLoader {
+  constructor(migrationDirectories, sortDirsSeparately, loadExtensions) {
+    this.sortDirsSeparately = sortDirsSeparately;
+
+    if (!Array.isArray(migrationDirectories)) {
+      migrationDirectories = [migrationDirectories];
+    }
+    this.migrationsPaths = migrationDirectories;
+    this.loadExtensions = loadExtensions || DEFAULT_LOAD_EXTENSIONS;
+  }
+
+  getFile(migrationsInfo) {
+    const absoluteDir = path.resolve(process.cwd(), migrationsInfo.directory);
+    const _path = path.join(absoluteDir, migrationsInfo.file);
+    const importFile = require('../util/import-file'); // late import
+    return importFile(_path);
+  }
+}
+
+module.exports = {
+  DEFAULT_LOAD_EXTENSIONS,
+  AbstractMigrationsLoader,
+};

lib/migrations/migrate/MigrationGenerator.js

@@ -1,6 +1,6 @@
 const path = require('path');
 const { writeJsFileUsingTemplate } = require('../util/template');
-const { getMergedConfig } = require('./configuration-merger');
+const { getMergedConfig } = require('./migrator-configuration-merger');
 const { ensureDirectoryExists } = require('../util/fs');
 const { yyyymmddhhmmss } = require('../util/timestamp');
 

lib/migrations/migrate/Migrator.js

@@ -4,7 +4,6 @@ const differenceWith = require('lodash/differenceWith');
 const get = require('lodash/get');
 const isEmpty = require('lodash/isEmpty');
 const max = require('lodash/max');
-const { inherits } = require('util');
 const {
   getLockTableName,
   getTable,
@@ -13,16 +12,16 @@ const {
 const { getSchemaBuilder } = require('./table-creator');
 const migrationListResolver = require('./migration-list-resolver');
 const MigrationGenerator = require('./MigrationGenerator');
-const { getMergedConfig } = require('./configuration-merger');
+const { getMergedConfig } = require('./migrator-configuration-merger');
 const { isBoolean, isFunction } = require('../../util/is');
 
-function LockError(msg) {
-  this.name = 'MigrationLocked';
-  this.message = msg;
+class LockError extends Error {
+  constructor(msg) {
+    super(msg);
+    this.name = 'MigrationLocked';
+  }
 }
 
-inherits(LockError, Error);
-
 // The new migration we're performing, typically called from the `knex.migrate`
 // interface on the main `knex` object. Passes the `knex` instance performing
 // the migration.

lib/migrations/migrate/configuration-merger.js → lib/migrations/migrate/migrator-configuration-merger.js

@@ -1,8 +1,6 @@
-const {
-  FsMigrations,
-  DEFAULT_LOAD_EXTENSIONS,
-} = require('./sources/fs-migrations');
+const { FsMigrations } = require('./sources/fs-migrations');
 const Logger = require('../../logger');
+const { DEFAULT_LOAD_EXTENSIONS } = require('../common/MigrationsLoader');
 const defaultLogger = new Logger();
 
 const CONFIG_DEFAULT = Object.freeze({

lib/migrations/migrate/sources/fs-migrations.js

@@ -2,30 +2,9 @@ const path = require('path');
 const sortBy = require('lodash/sortBy');
 
 const { readdir } = require('../../util/fs');
+const { AbstractMigrationsLoader } = require('../../common/MigrationsLoader');
 
-const DEFAULT_LOAD_EXTENSIONS = Object.freeze([
-  '.co',
-  '.coffee',
-  '.eg',
-  '.iced',
-  '.js',
-  '.cjs',
-  '.litcoffee',
-  '.ls',
-  '.ts',
-]);
-
-class FsMigrations {
-  constructor(migrationDirectories, sortDirsSeparately, loadExtensions) {
-    this.sortDirsSeparately = sortDirsSeparately;
-
-    if (!Array.isArray(migrationDirectories)) {
-      migrationDirectories = [migrationDirectories];
-    }
-    this.migrationsPaths = migrationDirectories;
-    this.loadExtensions = loadExtensions || DEFAULT_LOAD_EXTENSIONS;
-  }
-
+class FsMigrations extends AbstractMigrationsLoader {
   /**
    * Gets the migration names
    * @returns Promise<string[]>
@@ -77,11 +56,8 @@ class FsMigrations {
     return migration.file;
   }
 
-  getMigration(migration) {
-    const absoluteDir = path.resolve(process.cwd(), migration.directory);
-    const _path = path.join(absoluteDir, migration.file);
-    const importFile = require('../../util/import-file'); // late import
-    return importFile(_path);
+  getMigration(migrationInfo) {
+    return this.getFile(migrationInfo);
   }
 }
 
@@ -94,6 +70,5 @@ function filterMigrations(migrationSource, migrations, loadExtensions) {
 }
 
 module.exports = {
-  DEFAULT_LOAD_EXTENSIONS,
   FsMigrations,
 };

lib/migrations/seed/Seeder.js

@@ -2,91 +2,57 @@
 // -------
 
 const path = require('path');
-const flatten = require('lodash/flatten');
-const extend = require('lodash/extend');
-const includes = require('lodash/includes');
-const { ensureDirectoryExists, getFilepathsInFolder } = require('../util/fs');
+const { ensureDirectoryExists } = require('../util/fs');
 const { writeJsFileUsingTemplate } = require('../util/template');
 const { yyyymmddhhmmss } = require('../util/timestamp');
-
-const filterByLoadExtensions = (extensions) => (value) => {
-  const extension = path.extname(value);
-  return includes(extensions, extension);
-};
+const { getMergedConfig } = require('./seeder-configuration-merger');
 
 // The new seeds we're performing, typically called from the `knex.seed`
 // interface on the main `knex` object. Passes the `knex` instance performing
 // the seeds.
 class Seeder {
   constructor(knex) {
     this.knex = knex;
-    this.config = this.setConfig(knex.client.config.seeds);
+    this.config = this.resolveConfig(knex.client.config.seeds);
   }
 
   // Runs seed files for the given environment.
   async run(config) {
-    this.config = this.setConfig(config);
-    let files = await this._listAll();
-    if (config && config.specific) {
-      files = files.filter((file) => path.basename(file) === config.specific);
-      if (files.length === 0) {
-        throw new Error(
-          `Invalid argument provided: the specific seed "${config.specific}" does not exist.`
-        );
-      }
-    }
+    this.config = this.resolveConfig(config);
+    const files = await this.config.seedSource.getSeeds(this.config);
     return this._runSeeds(files);
   }
 
   // Creates a new seed file, with a given name.
   async make(name, config) {
-    this.config = this.setConfig(config);
+    this.config = this.resolveConfig(config);
     if (!name)
       throw new Error('A name must be specified for the generated seed');
     await this._ensureFolder(config);
     const seedPath = await this._writeNewSeed(name);
     return seedPath;
   }
 
-  // Lists all available seed files as a sorted array.
-  async _listAll(config) {
-    this.config = this.setConfig(config);
-    const { loadExtensions, recursive } = this.config;
-    const seeds = flatten(
-      await Promise.all(
-        this._absoluteConfigDirs().map((d) =>
-          getFilepathsInFolder(d, recursive)
-        )
-      )
-    );
-    // if true, each dir are already sorted
-    // (getFilepathsInFolderRecursively does this)
-    // if false, we need to sort all the seeds
-    if (this.config.sortDirsSeparately) {
-      return seeds.filter(filterByLoadExtensions(loadExtensions));
-    } else {
-      return seeds.filter(filterByLoadExtensions(loadExtensions)).sort();
-    }
-  }
-
   // Ensures a folder for the seeds exist, dependent on the
   // seed config settings.
   _ensureFolder() {
-    const dirs = this._absoluteConfigDirs();
+    const dirs = this.config.seedSource._getConfigDirectories(
+      this.config.logger
+    );
     const promises = dirs.map(ensureDirectoryExists);
     return Promise.all(promises);
   }
 
   // Run seed files, in sequence.
-  _runSeeds(seeds) {
-    seeds.forEach((seed) => this._validateSeedStructure(seed));
+  async _runSeeds(seeds) {
+    for (const seed of seeds) {
+      await this._validateSeedStructure(seed);
+    }
     return this._waterfallBatch(seeds);
   }
 
-  // Validates seed files by requiring and checking for a `seed` function.
   async _validateSeedStructure(filepath) {
-    const importFile = require('../util/import-file'); // late import
-    const seed = await importFile(filepath);
+    const seed = await this.config.seedSource.getSeed(filepath);
     if (typeof seed.seed !== 'function') {
       throw new Error(
         `Invalid seed file: ${filepath} must have a seed function`
@@ -114,7 +80,9 @@ class Seeder {
 
   _getNewStubFilePath(name) {
     const fileName = this._getNewStubFileName(name);
-    const dirs = this._absoluteConfigDirs();
+    const dirs = this.config.seedSource._getConfigDirectories(
+      this.config.logger
+    );
     const dir = dirs.slice(-1)[0]; // Get last specified directory
     return path.join(dir, fileName);
   }
@@ -132,13 +100,17 @@ class Seeder {
     return seedPath;
   }
 
+  async _listAll(config) {
+    this.config = this.resolveConfig(config);
+    return this.config.seedSource.getSeeds(this.config);
+  }
+
   // Runs a batch of seed files.
   async _waterfallBatch(seeds) {
     const { knex } = this;
     const log = [];
     for (const seedPath of seeds) {
-      const importFile = require('../util/import-file'); // late import
-      const seed = await importFile(seedPath);
+      const seed = await this.config.seedSource.getSeed(seedPath);
       try {
         await seed.seed(knex);
         log.push(seedPath);
@@ -157,47 +129,8 @@ class Seeder {
     return [log];
   }
 
-  /**
-   * Return all the config directories
-   * @returns {string[]}
-   */
-  _absoluteConfigDirs() {
-    const directories = Array.isArray(this.config.directory)
-      ? this.config.directory
-      : [this.config.directory];
-    return directories.map((directory) => {
-      if (!directory) {
-        console.warn(
-          'Failed to resolve config file, knex cannot determine where to run or make seeds'
-        );
-      }
-      return path.resolve(process.cwd(), directory);
-    });
-  }
-
-  setConfig(config) {
-    return extend(
-      {
-        extension: 'js',
-        directory: './seeds',
-        loadExtensions: [
-          '.co',
-          '.coffee',
-          '.eg',
-          '.iced',
-          '.js',
-          '.litcoffee',
-          '.ls',
-          '.ts',
-          '.cjs',
-        ],
-        timestampFilenamePrefix: false,
-        sortDirsSeparately: false,
-        recursive: false,
-      },
-      this.config || {},
-      config
-    );
+  resolveConfig(config) {
+    return getMergedConfig(config, this.config, this.knex.client.logger);
   }
 }
 

lib/migrations/seed/seeder-configuration-merger.js

@@ -0,0 +1,60 @@
+const { FsSeeds } = require('./sources/fs-seeds');
+const Logger = require('../../logger');
+const { DEFAULT_LOAD_EXTENSIONS } = require('../common/MigrationsLoader');
+const defaultLogger = new Logger();
+
+const CONFIG_DEFAULT = Object.freeze({
+  extension: 'js',
+  directory: './seeds',
+  loadExtensions: DEFAULT_LOAD_EXTENSIONS,
+  specific: null,
+  timestampFilenamePrefix: false,
+  recursive: false,
+  sortDirsSeparately: false,
+});
+
+function getMergedConfig(config, currentConfig, logger = defaultLogger) {
+  // config is the user specified config, mergedConfig has defaults and current config
+  // applied to it.
+  const mergedConfig = Object.assign(
+    {},
+    CONFIG_DEFAULT,
+    currentConfig || {},
+    config,
+    {
+      logger,
+    }
+  );
+
+  if (
+    config &&
+    // If user specifies any FS related config,
+    // clear specified migrationSource to avoid ambiguity
+    (config.directory ||
+      config.sortDirsSeparately !== undefined ||
+      config.loadExtensions)
+  ) {
+    if (config.seedSource) {
+      logger.warn(
+        'FS-related option specified for seed configuration. This resets seedSource to default FsMigrations'
+      );
+    }
+    mergedConfig.seedSource = null;
+  }
+
+  // If the user has not specified any configs, we need to
+  // default to fs migrations to maintain compatibility
+  if (!mergedConfig.seedSource) {
+    mergedConfig.seedSource = new FsSeeds(
+      mergedConfig.directory,
+      mergedConfig.sortDirsSeparately,
+      mergedConfig.loadExtensions
+    );
+  }
+
+  return mergedConfig;
+}
+
+module.exports = {
+  getMergedConfig,
+};