Support fresh subprocess for build backends (#3233)

docs/changelog/3227.feature.rst

@@ -0,0 +1 @@
+Support enabling fresh subprocess for packaging build backends via :ref:`fresh_subprocess` - by :user:`gaborbernat`.

docs/config.rst

@@ -794,6 +794,16 @@ Python virtual environment packaging
 
    Config settings (``dict[str, str]``) passed to the ``build_editable`` backend API endpoint.
 
+.. conf::
+   :keys: fresh_subprocess
+   :version_added: 4.14.0
+   :default: False
+
+   A flag controlling if each call to the build backend should be done in a fresh subprocess or not (especially older
+   build backends such as ``setuptools`` might require this to discover newly provisioned dependencies).
+
+   Directory where to put the project metadata files.
+
 Pip installer
 ~~~~~~~~~~~~~
 

src/tox/tox_env/python/virtual_env/package/pyproject.py

@@ -117,7 +117,8 @@ def id() -> str:
     @property
     def _frontend(self) -> Pep517VirtualEnvFrontend:
         if self._frontend_ is None:
-            self._frontend_ = Pep517VirtualEnvFrontend(self.root, self)
+            fresh = cast(bool, self.conf["fresh_subprocess"])
+            self._frontend_ = Pep517VirtualEnvFrontend(self.root, self, fresh_subprocess=fresh)
         return self._frontend_
 
     def register_config(self) -> None:
@@ -136,6 +137,12 @@ def register_config(self) -> None:
         )
         for key in ("sdist", "wheel", "editable"):
             self._add_config_settings(key)
+        self.conf.add_config(
+            keys=["fresh_subprocess"],
+            of_type=bool,
+            default=False,
+            desc="create a fresh subprocess for every backend request",
+        )
 
     def _add_config_settings(self, build_type: str) -> None:
         # config settings passed to PEP-517-compliant build backend https://peps.python.org/pep-0517/#config-settings
@@ -370,9 +377,10 @@ def id() -> str:
 
 
 class Pep517VirtualEnvFrontend(Frontend):
-    def __init__(self, root: Path, env: Pep517VenvPackager) -> None:
+    def __init__(self, root: Path, env: Pep517VenvPackager, *, fresh_subprocess: bool) -> None:
         super().__init__(*Frontend.create_args_from_folder(root))
         self._tox_env = env
+        self._fresh_subprocess = fresh_subprocess
         self._backend_executor_: LocalSubProcessPep517Executor | None = None
         into: dict[str, Any] = {}
 
@@ -412,19 +420,23 @@ def _send_msg(
         result_file: Path,  # noqa: ARG002
         msg: str,
     ) -> Iterator[ToxCmdStatus]:
-        with self._tox_env.execute_async(
-            cmd=self.backend_cmd,
-            cwd=self._root,
-            stdin=StdinSource.API,
-            show=None,
-            run_id=cmd,
-            executor=self.backend_executor,
-        ) as execute_status:
-            execute_status.write_stdin(f"{msg}{os.linesep}")
-            yield ToxCmdStatus(execute_status)
-        outcome = execute_status.outcome
-        if outcome is not None:  # pragma: no branch
-            outcome.assert_success()
+        try:
+            with self._tox_env.execute_async(
+                cmd=self.backend_cmd,
+                cwd=self._root,
+                stdin=StdinSource.API,
+                show=None,
+                run_id=cmd,
+                executor=self.backend_executor,
+            ) as execute_status:
+                execute_status.write_stdin(f"{msg}{os.linesep}")
+                yield ToxCmdStatus(execute_status)
+            outcome = execute_status.outcome
+            if outcome is not None:  # pragma: no branch
+                outcome.assert_success()
+        finally:
+            if self._fresh_subprocess:
+                self.backend_executor.close()
 
     def _unexpected_response(  # noqa: PLR0913
         self,

tox.ini

@@ -100,3 +100,4 @@ extras =
 commands =
     python -m pip list --format=columns
     python -c "print(r'{envpython}')"
+uv_seed = true