Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Remove unnecesary sleep when submitting parallel tasks #9097

Merged
merged 2 commits into from Apr 17, 2021
Merged

Remove unnecesary sleep when submitting parallel tasks #9097

Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
48c0c32
Remove unnecesary sleep when submitting parallel tasks
peterbell10 Apr 13, 2021
33b8f43
Remove redundant assignment
peterbell10 Apr 17, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
14 changes: 9 additions & 5 deletions sphinx/util/parallel.py
View file
Open in desktop
Expand Up @@ -100,12 +100,13 @@ def add_task(self, task_func: Callable, arg: Any = None, result_func: Callable =
args=(psend, task_func, arg))
self._procs[tid] = proc
self._precvsWaiting[tid] = precv
self._join_one()
_ = self._join_one()
tk0miya marked this conversation as resolved.
Show resolved Hide resolved

def join(self) -> None:
try:
while self._pworking:
self._join_one()
if not self._join_one():
time.sleep(0.02)
except Exception:
# shutdown other child processes on failure
self.terminate()
Expand All @@ -119,7 +120,8 @@ def terminate(self) -> None:
self._precvs.pop(tid)
self._pworking -= 1

def _join_one(self) -> None:
def _join_one(self) -> bool:
joined_any = False
for tid, pipe in self._precvs.items():
if pipe.poll():
exc, logs, result = pipe.recv()
Expand All @@ -131,15 +133,17 @@ def _join_one(self) -> None:
self._procs[tid].join()
self._precvs.pop(tid)
self._pworking -= 1
joined_any = True
break
else:
time.sleep(0.02)

while self._precvsWaiting and self._pworking < self.nproc:
newtid, newprecv = self._precvsWaiting.popitem()
self._precvs[newtid] = newprecv
self._procs[newtid].start()
self._pworking += 1

return joined_any


def make_chunks(arguments: Sequence[str], nproc: int, maxbatch: int = 10) -> List[Any]:
# determine how many documents to read in one go
Expand Down