feat(publish): Add just-in-time queue management

commands/publish/index.js

@@ -6,7 +6,6 @@ const crypto = require("crypto");
 const pFinally = require("p-finally");
 const pMap = require("p-map");
 const pPipe = require("p-pipe");
-const pReduce = require("p-reduce");
 const semver = require("semver");
 
 const Command = require("@lerna/command");
@@ -22,8 +21,7 @@ const npmPublish = require("@lerna/npm-publish");
 const packDirectory = require("@lerna/pack-directory");
 const logPacked = require("@lerna/log-packed");
 const { createRunner } = require("@lerna/run-lifecycle");
-const batchPackages = require("@lerna/batch-packages");
-const runParallelBatches = require("@lerna/run-parallel-batches");
+const runTopologically = require("@lerna/run-topologically");
 const pulseTillDone = require("@lerna/pulse-till-done");
 const versionCommand = require("@lerna/version");
 const prereleaseIdFromVersion = require("@lerna/prerelease-id-from-version");
@@ -164,16 +162,6 @@ class PublishCommand extends Command {
       this.updatesVersions = new Map(result.updatesVersions);
 
       this.packagesToPublish = this.updates.map(({ pkg }) => pkg).filter(pkg => !pkg.private);
-      this.batchedPackages = this.toposort
-        ? batchPackages(
-            this.packagesToPublish,
-            this.options.rejectCycles,
-            // Don't sort based on devDependencies because that
-            // would increase the chance of dependency cycles
-            // causing less-than-ideal a publishing order.
-            "dependencies"
-          )
-        : [this.packagesToPublish];
 
       if (result.needsConfirmation) {
         // only confirm for --canary, bump === "from-git",
@@ -575,6 +563,18 @@ class PublishCommand extends Command {
       });
   }
 
+  topoMapPackages(mapper) {
+    // we don't respect --no-sort here, sorry
+    return runTopologically(this.packagesToPublish, mapper, {
+      concurrency: this.concurrency,
+      rejectCycles: this.options.rejectCycles,
+      // Don't sort based on devDependencies because that
+      // would increase the chance of dependency cycles
+      // causing less-than-ideal a publishing order.
+      graphType: "dependencies",
+    });
+  }
+
   packUpdated() {
     const tracker = this.logger.newItem("npm pack");
 
@@ -616,9 +616,7 @@ class PublishCommand extends Command {
       ].filter(Boolean)
     );
 
-    chain = chain.then(() =>
-      pReduce(this.batchedPackages, (_, batch) => pMap(batch, mapper, { concurrency: 10 }))
-    );
+    chain = chain.then(() => this.topoMapPackages(mapper));
 
     chain = chain.then(() => removeTempLicenses(this.packagesToBeLicensed));
 
@@ -666,7 +664,7 @@ class PublishCommand extends Command {
       ].filter(Boolean)
     );
 
-    chain = chain.then(() => runParallelBatches(this.batchedPackages, this.concurrency, mapper));
+    chain = chain.then(() => this.topoMapPackages(mapper));
 
     if (!this.hasRootedLeaf) {
       // cyclical "publish" lifecycles are automatically skipped
@@ -709,7 +707,7 @@ class PublishCommand extends Command {
         });
     };
 
-    chain = chain.then(() => runParallelBatches(this.batchedPackages, this.concurrency, mapper));
+    chain = chain.then(() => this.topoMapPackages(mapper));
 
     return pFinally(chain, () => tracker.finish());
   }

commands/publish/package.json

@@ -34,7 +34,6 @@
     "test": "echo \"Run tests from root\" && exit 1"
   },
   "dependencies": {
-    "@lerna/batch-packages": "file:../../utils/batch-packages",
     "@lerna/check-working-tree": "file:../../utils/check-working-tree",
     "@lerna/child-process": "file:../../core/child-process",
     "@lerna/collect-updates": "file:../../utils/collect-updates",
@@ -50,7 +49,7 @@
     "@lerna/prompt": "file:../../core/prompt",
     "@lerna/pulse-till-done": "file:../../utils/pulse-till-done",
     "@lerna/run-lifecycle": "file:../../utils/run-lifecycle",
-    "@lerna/run-parallel-batches": "file:../../utils/run-parallel-batches",
+    "@lerna/run-topologically": "file:../../utils/run-topologically",
     "@lerna/validation-error": "file:../../core/validation-error",
     "@lerna/version": "file:../version",
     "figgy-pudding": "^3.5.1",
@@ -62,7 +61,6 @@
     "p-finally": "^1.0.0",
     "p-map": "^1.2.0",
     "p-pipe": "^1.2.0",
-    "p-reduce": "^1.0.0",
     "pacote": "^9.5.0",
     "semver": "^5.5.0"
   }