feat: add build concurrency control (close: #1819) (#2953)

* feat: add build concurrency control

* fix: rm comment

* feat: docs

Co-authored-by: genefy <ziheng.dzh@alibaba-inc.com>

Author: troyeagle

packages/@vuepress/core/lib/node/build/index.js

@@ -20,6 +20,7 @@ module.exports = class Build extends EventEmitter {
   constructor (context) {
     super()
     this.context = context
+    this.maxConcurrency = this.context.options.maxConcurrency
     this.outDir = this.context.outDir
   }
 
@@ -91,9 +92,18 @@ module.exports = class Build extends EventEmitter {
     // render pages
     logger.wait('Rendering static HTML...')
 
-    const pagePaths = await Promise.all(
-      this.context.pages.map(page => this.renderPage(page))
-    )
+    // If maxConcurrency is set, instead of rendering all pages concurrently,
+    // build task would render pages by smaller group to prevent OOM.
+    if (this.maxConcurrency) logger.info(`max concurrency set: ${this.maxConcurrency}`)
+    const pagePaths = []
+    const maxConcurrency = this.maxConcurrency || 100000
+    for (let i = 0; i < this.context.pages.length; i += maxConcurrency) {
+      const segmentPaths = await Promise.all(
+        this.context.pages.slice(i, i + maxConcurrency)
+        .map(page => this.renderPage(page))
+      )
+      pagePaths.push(...segmentPaths)
+    }
 
     readline.clearLine(process.stdout, 0)
     readline.cursorTo(process.stdout, 0)

packages/docs/docs/api/cli.md

@@ -54,6 +54,10 @@ Start development server in debug mode.
 
 Start development server in silent mode.
 
+#### --max-concurrency
+
+Set the max concurrency for rendering pages, to prevent OOM when rendering massive docs.
+
 ### dev
 
 Start a development server. All options from `vuepress build` are available. And there are several options specifically for dev:

packages/docs/docs/zh/api/cli.md

@@ -29,6 +29,9 @@ vuepress <command> targetDir [options]
 ### --silent
 以安静模式启动开发服务器。
 
+### --max-concurrency
+设置渲染文档的最大并发量，当渲染大量文档，可能造成内存溢出时使用
+
 ## dev
 
 启动一个开发服务器。来自 `vuepress build` 的所有选项都可用。除此以外，还有几个专门针对 dev 的选项：

packages/vuepress/lib/registerCoreCommands.js

@@ -51,6 +51,7 @@ module.exports = function (cli, options) {
     .option('--no-cache', 'clean the cache before build')
     .option('--debug', 'build in development mode for debugging')
     .option('--silent', 'build static site in silent mode')
+    .option('--max-concurrency', 'set the max docs concurrently processed when build static site')
     .action((sourceDir = '.', commandOptions) => {
       const { debug, silent } = commandOptions