Support providing a cwd for included tasks #110

README.rst

@@ -936,12 +936,12 @@ to use *fish* by default.
 Load tasks from another file
 ============================
 
-There are some scenarios where one might wish to define tasks outside of pyproject.toml.
-For example, if you want to share tasks between projects via git modules, generate tasks
-definitions dynamically, or simply have a lot of tasks and don't want the pyproject.toml
-to get too large. This can be achieved by creating a toml or json file within your
-project directory structure including the same structure for tasks as used in
-pyproject.toml
+There are some scenarios where one might wish to define tasks outside of pyproject.toml,
+or to collect tasks from multiple projects into one. For example, if you want to share
+tasks between projects via git modules, generate tasks definitions dynamically, organise
+your code in a monorepo, or simply have a lot of tasks and don't want the pyproject.toml
+to get too large. This can be achieved by creating a toml or json including the same
+structure for tasks as used in pyproject.toml
 
 For example:
 
@@ -972,6 +972,24 @@ so:
 Files are loaded in the order specified. If an item already exists then the included
 value it ignored.
 
+If an included task file itself includes other files, these second order includes are
+not inherited, so circular includes don't cause any problems.
+
+When including files from another location, you can also specify that tasks from that
+other file should be run from within a specific directory. For example with the
+following configuration, when tasks imported from my_subproject are run
+from the root, the task will actually execute as if it had been run from the
+my_subproject subdirectory.
+
+.. code-block:: toml
+
+  [[tool.poe.include]]
+  path = "my_subproject/pyproject.toml"
+  cwd  = "my_subproject"
+
+The cwd option still has the limitation that it cannot be used to specify a directory
+outside of parent directory of the pyproject.toml file that poe is running with.
+
 If a referenced file is missing then poe ignores it without error, though
 failure to read the contents will result in failure.
 

poethepoet/config.py

@@ -6,7 +6,7 @@
 except ImportError:
     import tomli  # type: ignore
 
-from typing import Any, Dict, Mapping, Optional, Sequence, Tuple, Union
+from typing import Any, Dict, List, Mapping, Optional, Sequence, Tuple, Union
 
 from .exceptions import PoeException
 
@@ -36,7 +36,7 @@ class PoeConfig:
         "env": dict,
         "envfile": (str, list),
         "executor": dict,
-        "include": (str, list),
+        "include": (str, list, dict),
         "poetry_command": str,
         "poetry_hooks": dict,
         "shell_interpreter": (str, list),
@@ -230,15 +230,31 @@ def find_pyproject_toml(self, target_dir: Optional[str] = None) -> Path:
         return maybe_result
 
     def _load_includes(self, project_dir: Path):
-        includes: Union[str, Sequence[str]] = self._poe.get("include", tuple())
-        if isinstance(includes, str):
-            includes = (includes,)
-
-        for path_str in includes:
-            if not isinstance(path_str, str):
-                raise PoeException(f"Invalid include value {path_str!r}")
+        include_option: Union[str, Sequence[str]] = self._poe.get("include", tuple())
+        includes: List[Dict[str, str]] = []
+
+        if isinstance(include_option, str):
+            includes.append({"path": include_option})
+        elif isinstance(include_option, dict):
+            includes.append(include_option)
+        elif isinstance(include_option, list):
+            valid_keys = {"path", "cwd"}
+            for include in include_option:
+                if isinstance(include, str):
+                    includes.append({"path": include})
+                elif (
+                    isinstance(include, dict)
+                    and include.get("path")
+                    and set(include.keys()) <= valid_keys
+                ):
+                    includes.append(include)
+                else:
+                    raise PoeException(
+                        f"Invalid item for the include option {include!r}"
+                    )
 
-            include_path = project_dir.joinpath(path_str).resolve()
+        for include in includes:
+            include_path = project_dir.joinpath(include["path"]).resolve()
 
             if not include_path.exists():
                 # TODO: print warning in verbose mode, requires access to ui somehow
@@ -247,7 +263,9 @@ def _load_includes(self, project_dir: Path):
             if include_path.name.endswith(".toml"):
                 try:
                     with include_path.open("rb") as file:
-                        self._merge_config(tomli.load(file), include_path)
+                        self._merge_config(
+                            tomli.load(file), include_path, include.get("cwd")
+                        )
                 except tomli.TOMLDecodeError as error:
                     raise PoeException(
                         f"Couldn't parse included toml file from {include_path}", error
@@ -256,21 +274,26 @@ def _load_includes(self, project_dir: Path):
             elif include_path.name.endswith(".json"):
                 try:
                     with include_path.open("rb") as file:
-                        self._merge_config(json.load(file), include_path)
+                        self._merge_config(
+                            json.load(file), include_path, include.get("cwd")
+                        )
                 except json.decoder.JSONDecodeError as error:
                     raise PoeException(
                         f"Couldn't parse included json file from {include_path}", error
                     ) from error
 
-    def _merge_config(self, extra_config: Mapping[str, Any], path: Path):
-        from .task import PoeTask
-
+    def _merge_config(
+        self,
+        extra_config: Mapping[str, Any],
+        include_path: Path,
+        task_cwd: Optional[str],
+    ):
         try:
             poe_config = extra_config.get("tool", {}).get("poe", {})
             tasks = poe_config.get("tasks", {})
         except AttributeError as error:
             raise PoeException(
-                f"Invalid content in included file from {path}", error
+                f"Invalid content in included file from {include_path}", error
             ) from error
 
         # Env is special because it can be extended rather than just overwritten
@@ -283,6 +306,22 @@ def _merge_config(self, extra_config: Mapping[str, Any], path: Path):
         # Includes additional tasks with preserved ordering
         self._poe["tasks"] = own_tasks = self._poe.get("tasks", {})
         for task_name, task_def in tasks.items():
+
+            if task_cwd:
+                # Override the config of each task to use the include level cwd as a
+                # base for the task level cwd
+                if "cwd" in task_def:
+                    # rebase the configured cwd onto the include level cwd
+                    task_def["cwd"] = (
+                        Path(self.project_dir)
+                        .joinpath(task_cwd)
+                        .resolve()
+                        .joinpath(task_def["cwd"])
+                        .relative_to(self.project_dir)
+                    )
+                else:
+                    task_def["cwd"] = task_cwd
+
             if task_name not in own_tasks:
                 own_tasks[task_name] = task_def
 

tests/fixtures/monorepo_project/pyproject.toml

@@ -0,0 +1,11 @@
+
+[[tool.poe.include]]
+path = "subproject_1/pyproject.toml"
+[[tool.poe.include]]
+path = "subproject_2/pyproject.toml"
+cwd  = "subproject_2"
+
+
+[tool.poe.tasks.get_cwd_0]
+interpreter = "python"
+shell       = "import os; print(os.getcwd())"

tests/fixtures/monorepo_project/subproject_1/pyproject.toml

@@ -0,0 +1,6 @@
+
+
+
+[tool.poe.tasks.get_cwd_1]
+interpreter = "python"
+shell       = "import os; print(os.getcwd())"

tests/fixtures/monorepo_project/subproject_2/extra_tasks.toml

@@ -0,0 +1,5 @@
+
+
+[tool.poe.tasks.extra_task]
+interpreter = "python"
+shell       = "print('nope')"

tests/fixtures/monorepo_project/subproject_2/pyproject.toml

@@ -0,0 +1,7 @@
+
+tool.poe.include = ["extra_tasks.toml", { path = "../pyproject.toml" }]
+
+[tool.poe.tasks.get_cwd_2]
+interpreter = "python"
+shell       = "import os; print(os.getcwd())"
+

tests/test_includes.py

@@ -67,7 +67,7 @@ def test_running_from_multiple_includes(run_poe_subproc, projects):
     assert result.stderr == ""
 
 
-def test_docs_for_onlyincludes(run_poe_subproc, projects):
+def test_docs_for_only_includes(run_poe_subproc, projects):
     result = run_poe_subproc(
         f'--root={projects["includes/only_includes"]}',
     )
@@ -80,3 +80,71 @@ def test_docs_for_onlyincludes(run_poe_subproc, projects):
     ) in result.capture
     assert result.stdout == ""
     assert result.stderr == ""
+
+
+def test_monorepo_contains_only_expected_tasks(run_poe_subproc, projects):
+    result = run_poe_subproc(project="monorepo")
+    assert result.capture.endswith(
+        "CONFIGURED TASKS\n"
+        "  get_cwd_0      \n"
+        "  get_cwd_1      \n"
+        "  get_cwd_2      \n\n\n"
+    )
+    assert result.stdout == ""
+    assert result.stderr == ""
+
+
+def test_monorepo_can_also_include_parent(run_poe_subproc, projects, is_windows):
+    result = run_poe_subproc(cwd=projects["monorepo/subproject_2"])
+    assert result.capture.endswith(
+        "CONFIGURED TASKS\n"
+        "  get_cwd_2      \n"
+        "  extra_task     \n"
+        "  get_cwd_0      \n\n\n"
+    )
+    assert result.stdout == ""
+    assert result.stderr == ""
+
+    result = run_poe_subproc("get_cwd_0", cwd=projects["monorepo/subproject_2"])
+    assert result.capture == "Poe => import os; print(os.getcwd())\n"
+    if is_windows:
+        assert result.stdout.endswith(
+            "poethepoet\\tests\\fixtures\\monorepo_project\\subproject_2\n"
+        )
+    else:
+        assert result.stdout.endswith(
+            "poethepoet/tests/fixtures/monorepo_project/subproject_2\n"
+        )
+    assert result.stderr == ""
+
+
+def test_monorepo_runs_each_task_with_expected_cwd(
+    run_poe_subproc, projects, is_windows
+):
+    result = run_poe_subproc("get_cwd_0", project="monorepo")
+    assert result.capture == "Poe => import os; print(os.getcwd())\n"
+    if is_windows:
+        assert result.stdout.endswith("poethepoet\\tests\\fixtures\\monorepo_project\n")
+    else:
+        assert result.stdout.endswith("poethepoet/tests/fixtures/monorepo_project\n")
+    assert result.stderr == ""
+
+    result = run_poe_subproc("get_cwd_1", project="monorepo")
+    assert result.capture == "Poe => import os; print(os.getcwd())\n"
+    if is_windows:
+        assert result.stdout.endswith("poethepoet\\tests\\fixtures\\monorepo_project\n")
+    else:
+        assert result.stdout.endswith("poethepoet/tests/fixtures/monorepo_project\n")
+    assert result.stderr == ""
+
+    result = run_poe_subproc("get_cwd_2", project="monorepo")
+    assert result.capture == "Poe => import os; print(os.getcwd())\n"
+    if is_windows:
+        assert result.stdout.endswith(
+            "poethepoet\\tests\\fixtures\\monorepo_project\\subproject_2\n"
+        )
+    else:
+        assert result.stdout.endswith(
+            "poethepoet/tests/fixtures/monorepo_project/subproject_2\n"
+        )
+    assert result.stderr == ""