From 882b69cc7c4735e735b6b4d308d72e1dc3b90904 Mon Sep 17 00:00:00 2001 From: Robbie McKinstry Date: Fri, 21 Oct 2022 14:45:04 -0400 Subject: [PATCH 1/3] Configure default executor to use --parallel setting. --- ...n--python-runtime-now-respects-parallel-flag.yaml | 4 ++++ sdk/python/cmd/pulumi-language-python-exec | 12 ++++++++++++ sdk/python/lib/pulumi/runtime/settings.py | 7 +++---- 3 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 changelog/pending/20221022--sdk-python--python-runtime-now-respects-parallel-flag.yaml diff --git a/changelog/pending/20221022--sdk-python--python-runtime-now-respects-parallel-flag.yaml b/changelog/pending/20221022--sdk-python--python-runtime-now-respects-parallel-flag.yaml new file mode 100644 index 000000000000..280f69c2faaa --- /dev/null +++ b/changelog/pending/20221022--sdk-python--python-runtime-now-respects-parallel-flag.yaml @@ -0,0 +1,4 @@ +changes: +- type: fix + scope: sdk/python + description: Python runtime now respects the --parallel flag. diff --git a/sdk/python/cmd/pulumi-language-python-exec b/sdk/python/cmd/pulumi-language-python-exec index f49495884217..c40be7a5f4ee 100755 --- a/sdk/python/cmd/pulumi-language-python-exec +++ b/sdk/python/cmd/pulumi-language-python-exec @@ -3,11 +3,13 @@ import argparse import asyncio +from typing import Optional import logging import os import sys import traceback import runpy +from concurrent.futures import ThreadPoolExecutor # The user might not have installed Pulumi yet in their environment - provide a high-quality error message in that case. try: @@ -51,6 +53,13 @@ def _get_user_stacktrace(user_program_abspath: str) -> str: # we did not detect a __main__ program, return normal traceback return traceback.format_exc() +def _set_default_executor(loop, parallelism: Optional[int]): + '''configure this event loop to respect the settings provided.''' + if parallelism is None: + return + exec = ThreadPoolExecutor(max_workers=parallelism) + loop.set_default_executor(exec) + if __name__ == "__main__": # Parse the arguments, program name, and optional arguments. ap = argparse.ArgumentParser(description='Execute a Pulumi Python program') @@ -125,6 +134,9 @@ if __name__ == "__main__": except RuntimeError: loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) + + # Configure the event loop to respect the parallelism value provided as input. + _set_default_executor(loop, settings.parallel) # We are (unfortunately) suppressing the log output of asyncio to avoid showing to users some of the bad things we # do in our programming model. diff --git a/sdk/python/lib/pulumi/runtime/settings.py b/sdk/python/lib/pulumi/runtime/settings.py index 17bb9b569c5f..ed8aa1e59fe8 100644 --- a/sdk/python/lib/pulumi/runtime/settings.py +++ b/sdk/python/lib/pulumi/runtime/settings.py @@ -208,10 +208,6 @@ def get_engine() -> Optional[Union[engine_pb2_grpc.EngineStub, Any]]: """ return SETTINGS.engine - -ROOT: ContextVar[Optional[Resource]] = ContextVar("root_resource", default=None) - - def get_root_resource() -> Optional["Resource"]: """ Returns the implicit root stack resource for all resources created in this program. @@ -226,6 +222,9 @@ def set_root_resource(root: "Resource"): ROOT.set(root) +ROOT: ContextVar[Optional[Resource]] = ContextVar("root_resource", default=None) + + async def monitor_supports_feature(feature: str) -> bool: if feature not in SETTINGS.feature_support: monitor = SETTINGS.monitor From 59a5c61f57171997f7a0483c1359bcc24ff619f5 Mon Sep 17 00:00:00 2001 From: Robbie McKinstry Date: Sat, 22 Oct 2022 18:33:07 -0400 Subject: [PATCH 2/3] Format file --- sdk/python/lib/pulumi/runtime/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/lib/pulumi/runtime/settings.py b/sdk/python/lib/pulumi/runtime/settings.py index ed8aa1e59fe8..efdb2ade788e 100644 --- a/sdk/python/lib/pulumi/runtime/settings.py +++ b/sdk/python/lib/pulumi/runtime/settings.py @@ -208,6 +208,7 @@ def get_engine() -> Optional[Union[engine_pb2_grpc.EngineStub, Any]]: """ return SETTINGS.engine + def get_root_resource() -> Optional["Resource"]: """ Returns the implicit root stack resource for all resources created in this program. From 36e2f1977a744e4cd9aa6cb8ab4b0dc9afaa2c6d Mon Sep 17 00:00:00 2001 From: Robbie McKinstry Date: Sat, 22 Oct 2022 18:54:31 -0400 Subject: [PATCH 3/3] Parallelism must be at least 1. --- sdk/python/cmd/pulumi-language-python-exec | 1 + 1 file changed, 1 insertion(+) diff --git a/sdk/python/cmd/pulumi-language-python-exec b/sdk/python/cmd/pulumi-language-python-exec index c40be7a5f4ee..ae522177e172 100755 --- a/sdk/python/cmd/pulumi-language-python-exec +++ b/sdk/python/cmd/pulumi-language-python-exec @@ -57,6 +57,7 @@ def _set_default_executor(loop, parallelism: Optional[int]): '''configure this event loop to respect the settings provided.''' if parallelism is None: return + parallelism = max(parallelism, 1) exec = ThreadPoolExecutor(max_workers=parallelism) loop.set_default_executor(exec)