From b78186d3b5bbb895bdb4f10a98db7ad4dea71298 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Mon, 16 May 2022 02:16:57 +0900 Subject: [PATCH 1/3] Fix #9096: sphinx-build: the value of progress bar for paralle build is wrong The value of progress bar (reading and writing) should be increased just after each task is finished. --- CHANGES | 1 + sphinx/builders/__init__.py | 32 +++++++++++++++++++++++--------- sphinx/util/__init__.py | 17 +++++++++++------ 3 files changed, 35 insertions(+), 15 deletions(-) diff --git a/CHANGES b/CHANGES index f65b0b07ab2..240f12e1066 100644 --- a/CHANGES +++ b/CHANGES @@ -39,6 +39,7 @@ Bugs fixed * #9575: autodoc: The annotation of return value should not be shown when ``autodoc_typehints="description"`` +* #9096: sphinx-build: the value of progress bar for paralle build is wrong Testing -------- diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 45188cd565c..4936ecf368c 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -25,6 +25,7 @@ from sphinx.util.osutil import SEP, ensuredir, relative_uri, relpath from sphinx.util.parallel import ParallelTasks, SerialTasks, make_chunks, parallel_available from sphinx.util.tags import Tags +from sphinx.util.typing import NoneType # side effect: registers roles and directives from sphinx import directives # NOQA isort:skip @@ -429,6 +430,13 @@ def _read_serial(self, docnames: List[str]) -> None: self.read_doc(docname) def _read_parallel(self, docnames: List[str], nproc: int) -> None: + chunks = make_chunks(docnames, nproc) + + # create a statas_iterator to step progressbar after reading a document + # (see: ``merge()`` function) + progress = status_iterator(chunks, __('reading sources... '), "purple", + len(chunks), self.app.verbosity) + # clear all outdated docs at once for docname in docnames: self.events.emit('env-purge-doc', self.env, docname) @@ -445,16 +453,15 @@ def merge(docs: List[str], otherenv: bytes) -> None: env = pickle.loads(otherenv) self.env.merge_info_from(docs, env, self.app) - tasks = ParallelTasks(nproc) - chunks = make_chunks(docnames, nproc) + next(progress) - for chunk in status_iterator(chunks, __('reading sources... '), "purple", - len(chunks), self.app.verbosity): + tasks = ParallelTasks(nproc) + for chunk in chunks: tasks.add_task(read_process, chunk, merge) # make sure all threads have finished - logger.info(bold(__('waiting for workers...'))) tasks.join() + logger.info('') def read_doc(self, docname: str) -> None: """Parse a file and add/update inventory entries for the doctree.""" @@ -563,19 +570,26 @@ def write_process(docs: List[Tuple[str, nodes.document]]) -> None: tasks = ParallelTasks(nproc) chunks = make_chunks(docnames, nproc) + # create a statas_iterator to step progressbar after reading a document + # (see: ``on_chunk_done()`` function) + progress = status_iterator(chunks, __('reading sources... '), "purple", + len(chunks), self.app.verbosity) + + def on_chunk_done(args: List[Tuple[str, NoneType]], result: NoneType) -> None: + next(progress) + self.app.phase = BuildPhase.RESOLVING - for chunk in status_iterator(chunks, __('writing output... '), "darkgreen", - len(chunks), self.app.verbosity): + for chunk in chunks: arg = [] for docname in chunk: doctree = self.env.get_and_resolve_doctree(docname, self) self.write_doc_serialized(docname, doctree) arg.append((docname, doctree)) - tasks.add_task(write_process, arg) + tasks.add_task(write_process, arg, on_chunk_done) # make sure all threads have finished - logger.info(bold(__('waiting for workers...'))) tasks.join() + logger.info('') def prepare_writing(self, docnames: Set[str]) -> None: """A place where you can add logic before :meth:`write_doc` is run""" diff --git a/sphinx/util/__init__.py b/sphinx/util/__init__.py index cd65b94b966..b5a49a9d118 100644 --- a/sphinx/util/__init__.py +++ b/sphinx/util/__init__.py @@ -13,8 +13,8 @@ from importlib import import_module from os import path from time import mktime, strptime -from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Iterable, Iterator, List, Optional, - Pattern, Set, Tuple, Type) +from typing import (IO, TYPE_CHECKING, Any, Callable, Dict, Generator, Iterable, List, + Optional, Pattern, Set, Tuple, Type, TypeVar) from urllib.parse import parse_qsl, quote_plus, urlencode, urlsplit, urlunsplit from sphinx.errors import ExtensionError, FiletypeNotFoundError, SphinxParallelError @@ -445,8 +445,12 @@ def display_chunk(chunk: Any) -> str: return str(chunk) -def old_status_iterator(iterable: Iterable, summary: str, color: str = "darkgreen", - stringify_func: Callable[[Any], str] = display_chunk) -> Iterator: +T = TypeVar('T') + + +def old_status_iterator(iterable: Iterable[T], summary: str, color: str = "darkgreen", + stringify_func: Callable[[Any], str] = display_chunk + ) -> Generator[T, None, None]: l = 0 for item in iterable: if l == 0: @@ -460,9 +464,10 @@ def old_status_iterator(iterable: Iterable, summary: str, color: str = "darkgree # new version with progress info -def status_iterator(iterable: Iterable, summary: str, color: str = "darkgreen", +def status_iterator(iterable: Iterable[T], summary: str, color: str = "darkgreen", length: int = 0, verbosity: int = 0, - stringify_func: Callable[[Any], str] = display_chunk) -> Iterable: + stringify_func: Callable[[Any], str] = display_chunk + ) -> Generator[T, None, None]: if length == 0: yield from old_status_iterator(iterable, summary, color, stringify_func) return From 41426431cf54bc9d0bcd52251abec2a9d36bdbc5 Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 22 May 2022 15:19:13 +0900 Subject: [PATCH 2/3] doc: Fix copy-paste error Co-authored-by: peterbell10 --- sphinx/builders/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 4936ecf368c..316fe19132b 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -570,9 +570,9 @@ def write_process(docs: List[Tuple[str, nodes.document]]) -> None: tasks = ParallelTasks(nproc) chunks = make_chunks(docnames, nproc) - # create a statas_iterator to step progressbar after reading a document + # create a statas_iterator to step progressbar after writing a document # (see: ``on_chunk_done()`` function) - progress = status_iterator(chunks, __('reading sources... '), "purple", + progress = status_iterator(chunks, __('writing output... '), "darkgreen", len(chunks), self.app.verbosity) def on_chunk_done(args: List[Tuple[str, NoneType]], result: NoneType) -> None: From 7e031ab6cf152e48ec00e53b4fe8f322cbf6006e Mon Sep 17 00:00:00 2001 From: Takeshi KOMIYA Date: Sun, 22 May 2022 15:21:35 +0900 Subject: [PATCH 3/3] Fix typo --- sphinx/builders/__init__.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sphinx/builders/__init__.py b/sphinx/builders/__init__.py index 316fe19132b..d8500e11b42 100644 --- a/sphinx/builders/__init__.py +++ b/sphinx/builders/__init__.py @@ -432,7 +432,7 @@ def _read_serial(self, docnames: List[str]) -> None: def _read_parallel(self, docnames: List[str], nproc: int) -> None: chunks = make_chunks(docnames, nproc) - # create a statas_iterator to step progressbar after reading a document + # create a status_iterator to step progressbar after reading a document # (see: ``merge()`` function) progress = status_iterator(chunks, __('reading sources... '), "purple", len(chunks), self.app.verbosity) @@ -570,7 +570,7 @@ def write_process(docs: List[Tuple[str, nodes.document]]) -> None: tasks = ParallelTasks(nproc) chunks = make_chunks(docnames, nproc) - # create a statas_iterator to step progressbar after writing a document + # create a status_iterator to step progressbar after writing a document # (see: ``on_chunk_done()`` function) progress = status_iterator(chunks, __('writing output... '), "darkgreen", len(chunks), self.app.verbosity)