@lerna/run with just-in-time queue management

commands/run/index.js

@@ -1,13 +1,13 @@
 "use strict";
 
 const pMap = require("p-map");
+const PQueue = require("p-queue");
 
 const Command = require("@lerna/command");
 const npmRunScript = require("@lerna/npm-run-script");
-const batchPackages = require("@lerna/batch-packages");
-const runParallelBatches = require("@lerna/run-parallel-batches");
 const output = require("@lerna/output");
 const timer = require("@lerna/timer");
+const QueryGraph = require("@lerna/query-graph");
 const ValidationError = require("@lerna/validation-error");
 const { getFilteredPackages } = require("@lerna/filter-options");
 
@@ -58,10 +58,6 @@ class RunCommand extends Command {
         // still exits zero, aka "ok"
         return false;
       }
-
-      this.batchedPackages = this.toposort
-        ? batchPackages(this.packagesWithScript, this.options.rejectCycles)
-        : [this.packagesWithScript];
     });
   }
 
@@ -80,7 +76,7 @@ class RunCommand extends Command {
     if (this.options.parallel) {
       chain = chain.then(() => this.runScriptInPackagesParallel());
     } else {
-      chain = chain.then(() => this.runScriptInPackagesBatched());
+      chain = chain.then(() => this.runScriptInPackagesTopological());
     }
 
     if (this.bail) {
@@ -130,14 +126,35 @@ class RunCommand extends Command {
     };
   }
 
-  runScriptInPackagesBatched() {
+  runScriptInPackagesTopological() {
+    const queue = new PQueue({ concurrency: this.concurrency });
+    const graph = new QueryGraph(this.packagesWithScript, this.options.rejectCycles);
+
     const runner = this.options.stream
       ? pkg => this.runScriptInPackageStreaming(pkg)
       : pkg => this.runScriptInPackageCapturing(pkg);
 
-    return runParallelBatches(this.batchedPackages, this.concurrency, runner).then(batchedResults =>
-      batchedResults.reduce((arr, batch) => arr.concat(batch), [])
-    );
+    return new Promise((resolve, reject) => {
+      const returnValues = [];
+
+      const queueNextAvailablePackages = () =>
+        graph.getAvailablePackages().forEach(({ pkg, name }) => {
+          graph.markAsTaken(name);
+
+          queue
+            .add(() =>
+              runner(pkg)
+                .then(value => returnValues.push(value))
+                .then(() => graph.markAsDone(pkg))
+                .then(() => queueNextAvailablePackages())
+            )
+            .catch(reject);
+        });
+
+      queueNextAvailablePackages();
+
+      return queue.onIdle().then(() => resolve(returnValues));
+    });
   }
 
   runScriptInPackagesParallel() {

commands/run/package.json

@@ -35,14 +35,14 @@
     "populate--": true
   },
   "dependencies": {
-    "@lerna/batch-packages": "file:../../utils/batch-packages",
     "@lerna/command": "file:../../core/command",
     "@lerna/filter-options": "file:../../core/filter-options",
     "@lerna/npm-run-script": "file:../../utils/npm-run-script",
     "@lerna/output": "file:../../utils/output",
-    "@lerna/run-parallel-batches": "file:../../utils/run-parallel-batches",
+    "@lerna/query-graph": "file:../../utils/query-graph",
     "@lerna/timer": "file:../../utils/timer",
     "@lerna/validation-error": "file:../../core/validation-error",
-    "p-map": "^1.2.0"
+    "p-map": "^1.2.0",
+    "p-queue": "^4.0.0"
   }
 }

core/package-graph/index.js

@@ -188,9 +188,10 @@ class PackageGraph extends Map {
 
   /**
    * Return a tuple of cycle paths and nodes, which have been removed from the graph.
+   * @param {!boolean} pruneCycles Prune graph of cycle nodes
    * @returns [Set<String[]>, Set<PackageGraphNode>]
    */
-  partitionCycles() {
+  partitionCycles(pruneCycles = true) {
     const cyclePaths = new Set();
     const cycleNodes = new Set();
 
@@ -234,7 +235,7 @@ class PackageGraph extends Map {
       currentNode.localDependents.forEach(visits([currentName]));
     });
 
-    if (cycleNodes.size) {
+    if (cycleNodes.size && pruneCycles) {
       this.prune(...cycleNodes);
     }
 

utils/query-graph/README.md

@@ -0,0 +1,9 @@
+# `@lerna/query-graph`
+
+> An internal Lerna tool
+
+## Usage
+
+You probably shouldn't, at least directly.
+
+Install [lerna](https://www.npmjs.com/package/lerna) for access to the `lerna` CLI.