diff --git a/CHANGES b/CHANGES index 4cb10f8a57e..2abf528a5a9 100644 --- a/CHANGES +++ b/CHANGES @@ -23,6 +23,7 @@ Bugs fixed * #10443: epub: EPUB builder can't detect the mimetype of .webp file * #10456: py domain: ``:meta:`` fields are displayed if docstring contains two or more meta-field +* #9096: sphinx-build: the value of progress bar for paralle build is wrong Testing -------- diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 45188cd565c..d8500e11b42 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -25,6 +25,7 @@ from sphinx.util.osutil import SEP, ensuredir, relative_uri, relpath from sphinx.util.parallel import ParallelTasks, SerialTasks, make_chunks, parallel_available from sphinx.util.tags import Tags +from sphinx.util.typing import NoneType # side effect: registers roles and directives from sphinx import directives # NOQA isort:skip @@ -429,6 +430,13 @@ def _read_serial(self, docnames: List[str]) -> None: self.read_doc(docname) def _read_parallel(self, docnames: List[str], nproc: int) -> None: + chunks = make_chunks(docnames, nproc) + + # create a status_iterator to step progressbar after reading a document + # (see: ``merge()`` function) + progress = status_iterator(chunks, __('reading sources... '), "purple", + len(chunks), self.app.verbosity) + # clear all outdated docs at once for docname in docnames: self.events.emit('env-purge-doc', self.env, docname) @@ -445,16 +453,15 @@ def merge(docs: List[str], otherenv: bytes) -> None: env = pickle.loads(otherenv) self.env.merge_info_from(docs, env, self.app) - tasks = ParallelTasks(nproc) - chunks = make_chunks(docnames, nproc) + next(progress) - for chunk in status_iterator(chunks, __('reading sources... '), "purple", - len(chunks), self.app.verbosity): + tasks = ParallelTasks(nproc) + for chunk in chunks: tasks.add_task(read_process, chunk, merge) # make sure all threads have finished - logger.info(bold(__('waiting for workers...'))) tasks.join() + logger.info('') def read_doc(self, docname: str) -> None: """Parse a file and add/update inventory entries for the doctree.""" @@ -563,19 +570,26 @@ def write_process(docs: List[Tuple[str, nodes.document]]) -> None: tasks = ParallelTasks(nproc) chunks = make_chunks(docnames, nproc) + # create a status_iterator to step progressbar after writing a document + # (see: ``on_chunk_done()`` function) + progress = status_iterator(chunks, __('writing output... '), "darkgreen", + len(chunks), self.app.verbosity) + + def on_chunk_done(args: List[Tuple[str, NoneType]], result: NoneType) -> None: + next(progress) + self.app.phase = BuildPhase.RESOLVING - for chunk in status_iterator(chunks, __('writing output... '), "darkgreen", - len(chunks), self.app.verbosity): + for chunk in chunks: arg = [] for docname in chunk: doctree = self.env.get_and_resolve_doctree(docname, self) self.write_doc_serialized(docname, doctree) arg.append((docname, doctree)) - tasks.add_task(write_process, arg) + tasks.add_task(write_process, arg, on_chunk_done) # make sure all threads have finished - logger.info(bold(__('waiting for workers...'))) tasks.join() + logger.info('') def prepare_writing(self, docnames: Set[str]) -> None: """A place where you can add logic before :meth:`write_doc` is run""" diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index cd65b94b966..b5a49a9d118 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -13,8 +13,8 @@ from importlib import import_module from os import path from time import mktime, strptime -from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator, List, Optional, - Pattern, Set, Tuple, Type) +from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, List, + Optional, Pattern, Set, Tuple, Type, TypeVar) from urllib.parse import parse_qsl, quote_plus, urlencode, urlsplit, urlunsplit from sphinx.errors import ExtensionError, FiletypeNotFoundError, SphinxParallelError @@ -445,8 +445,12 @@ def display_chunk(chunk: Any) -> str: return str(chunk) -def old_status_iterator(iterable: Iterable, summary: str, color: str = "darkgreen", - stringify_func: Callable[[Any], str] = display_chunk) -> Iterator: +T = TypeVar('T') + + +def old_status_iterator(iterable: Iterable[T], summary: str, color: str = "darkgreen", + stringify_func: Callable[[Any], str] = display_chunk + ) -> Generator[T, None, None]: l = 0 for item in iterable: if l == 0: @@ -460,9 +464,10 @@ def old_status_iterator(iterable: Iterable, summary: str, color: str = "darkgree # new version with progress info -def status_iterator(iterable: Iterable, summary: str, color: str = "darkgreen", +def status_iterator(iterable: Iterable[T], summary: str, color: str = "darkgreen", length: int = 0, verbosity: int = 0, - stringify_func: Callable[[Any], str] = display_chunk) -> Iterable: + stringify_func: Callable[[Any], str] = display_chunk + ) -> Generator[T, None, None]: if length == 0: yield from old_status_iterator(iterable, summary, color, stringify_func) return