.pre-commit-config.yaml

@@ -34,16 +34,16 @@ repos:
       - id: black
         args: [--safe]
   - repo: https://github.com/asottile/blacken-docs
-    rev: v1.12.1
+    rev: 1.13.0
     hooks:
       - id: blacken-docs
         additional_dependencies: [black==22.12]
   - repo: https://github.com/pre-commit/pygrep-hooks
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
       - id: rst-backticks
   - repo: https://github.com/tox-dev/tox-ini-fmt
-    rev: "0.6.0"
+    rev: "0.6.1"
     hooks:
       - id: tox-ini-fmt
         args: ["-p", "fix"]
@@ -61,7 +61,7 @@ repos:
           - pep8-naming==0.13.3
           - flake8-pyproject==1.2.2
   - repo: https://github.com/pre-commit/mirrors-prettier
-    rev: "v2.7.1"
+    rev: "v3.0.0-alpha.4"
     hooks:
       - id: prettier
         additional_dependencies:

docs/changelog.rst

@@ -4,6 +4,45 @@ Release History
 
 .. towncrier release notes start
 
+v4.4.0 (2023-01-25)
+-------------------
+
+Features - 4.4.0
+~~~~~~~~~~~~~~~~
+- Test environments now recognize boolean config keys ``constrain_package_deps`` (default=true) and ``use_frozen_constraints`` (default=false),
+  which control how tox generates and applies constraints files when performing ``install_package_deps``.
+
+  If ``constrain_package_deps`` is true (default), then tox will write out ``{env_dir}{/}constraints.txt`` and pass it to
+  ``pip`` during ``install_package_deps``. If ``use_frozen_constraints`` is false (default), the constraints will be taken
+  from the specifications listed under ``deps`` (and inside any requirements or constraints file referenced in ``deps``).
+  Otherwise, ``list_dependencies_command`` (``pip freeze``) is used to enumerate exact package specifications which will
+  be written to the constraints file.
+
+  In previous releases, conflicting package dependencies would silently override the ``deps`` named in the configuration,
+  resulting in test runs against unexpected dependency versions, particularly when using tox factors to explicitly test
+  with different versions of dependencies - by :user:`masenf`. (:issue:`2386`)
+
+Bugfixes - 4.4.0
+~~~~~~~~~~~~~~~~
+- When parsing command lines, use ``shlex(..., posix=True)``, even on windows platforms, since non-POSIX mode does not
+  handle escape characters and quoting like a shell would. This improves cross-platform configurations without hacks or
+  esoteric quoting.
+
+  To make this transition easier, on Windows, the backslash path separator will not treated as an escape character unless
+  it preceeds a quote, whitespace, or another backslash chracter. This allows paths to mostly be written in single or
+  double backslash style.
+
+  Note that **double-backslash will no longer be escaped to a single backslash in substitutions**, instead the double
+  backslash will be consumed as part of command splitting, on either posix or windows platforms.
+
+  In some instances superfluous double or single quote characters may be stripped from arg arrays in ways that do not
+  occur in the default windows ``cmd.exe`` shell - by :user:`masenf`. (:issue:`2635`)
+
+Improved Documentation - 4.4.0
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+- Add infromation when command from ``list_dependencies_command`` configuration option is used. (:issue:`2883`)
+
+
 v4.3.5 (2023-01-18)
 -------------------
 

docs/config.rst

@@ -431,6 +431,16 @@ Run
       that outside of it. Therefore ``python`` translates as the virtual environments ``python`` (having the same
       runtime version as the :ref:`base_python`), and ``pip`` translates as the virtual environments ``pip``.
 
+   .. note::
+
+       ``shlex`` POSIX-mode quoting rules are used to split the command line into arguments on all
+       supported platforms as of tox 4.4.0.
+
+       The backslash ``\`` character can be used to escape quotes, whitespace, itself, and
+       other characters (except on Windows, where a backslash in a path will not be interpreted as an escape).
+       Unescaped single quote will disable the backslash escape until closed by another unescaped single quote.
+       For more details, please see :doc:`shlex parsing rules <python:library/shlex>`.
+
    .. note::
 
      Inline scripts can be used, however note these are discovered from the project root directory, and is not
@@ -742,6 +752,8 @@ Pip installer
    :version_added: 2.4
 
    The ``list_dependencies_command`` setting is used for listing the packages installed into the virtual environment.
+   This command will be executed only if executing on Contionous Integrations is detected (for example set environment variable ``CI=1``)
+   or if journal is active.
 
 
 .. conf::
@@ -753,6 +765,25 @@ Pip installer
    latest available pre-release of any dependencies without a specified version. If ``false``, pip will only install
    final releases of unpinned dependencies.
 
+.. conf::
+   :keys: constrain_package_deps
+   :default: true
+   :version_added: 4.4.0
+
+   If ``constrain_package_deps`` is true, then tox will create and use ``{env_dir}{/}constraints.txt`` when installing
+   package dependnecies during ``install_package_deps`` stage. When this value is set to false, any conflicting package
+   dependencies will override explicit dependencies and constraints passed to ``deps``.
+
+.. conf::
+   :keys: use_frozen_constraints
+   :default: false
+   :version_added: 4.4.0
+
+   When ``use_frozen_constraints`` is true, then tox will use the ``list_dependencies_command`` to enumerate package
+   versions in order to create ``{env_dir}{/}constraints.txt``. Otherwise the package specifications explicitly listed under
+   ``deps`` (or in requirements / constraints files referenced in ``deps``) will be used as the constraints. If
+   ``constrain_package_deps`` is false, then this setting has no effect.
+
 User configuration
 ------------------
 
@@ -774,14 +805,18 @@ through the ``{...}`` string-substitution pattern.
 
 The string inside the curly braces may reference a global or per-environment config key as described above.
 
-The backslash character ``\`` will act as an escape for a following: ``\``,
+In substitutions, the backslash character ``\`` will act as an escape when preceeding
 ``{``, ``}``, ``:``, ``[``, or ``]``, otherwise the backslash will be
 reproduced literally::
 
     commands =
         python -c 'print("\{posargs} = \{}".format("{posargs}"))'
         python -c 'print("host: \{}".format("{env:HOSTNAME:host\: not set}")'
 
+Note that any backslashes remaining after substitution may be processed by ``shlex`` during command parsing. On POSIX
+platforms, the backslash will escape any following character; on windows, the backslash will escape any following quote,
+whitespace, or backslash character (since it normally acts as a path delimiter).
+
 Special substitutions that accept additional colon-delimited ``:`` parameters
 cannot have a space after the ``:`` at the beginning of line (e.g.  ``{posargs:
 magic}`` would be parsed as factorial ``{posargs``, having value magic).

docs/faq.rst

@@ -124,21 +124,16 @@ install. While creating a test environment tox will invoke pip multiple times, i
    1. install the dependencies of the package.
    2. install the package itself.
 
-Some solutions and their drawbacks:
-
-- specify the constraint files within :ref:`deps` (these constraints will not be applied when installing package
-  dependencies),
-- use ``PIP_CONSTRAINT`` inside :ref:`set_env` (tox will not know about the content of the constraint file and such
-  will not trigger a rebuild of the environment when its content changes),
-- specify the constraint file by extending the :ref:`install_command` as in the following example
-  (tox will not know about the content of the constraint file and such will not trigger a rebuild of the environment
-  when its content changes).
-
-.. code-block:: ini
-
-    [testenv:py39]
-    install_command = python -m pip install {opts} {packages} -c constraints.txt
-    extras = test
+Starting in tox 4.4.0, ``{env_dir}{/}constraints.txt`` is generated by default during ``install_deps`` based on the
+package specifications listed under ``deps``. These constraints are subsequently passed to pip during the
+``install_package_deps`` stage, causing an error to be raised when the package dependencies conflict with the test
+environment dependencies. For stronger guarantees, set ``use_frozen_constraints = true`` in the test environment to
+generate the constraints file based on the exact versions enumerated by the ``list_dependencies_command`` (``pip
+freeze``).  When using frozen constraints, if the package deps are incompatible with any previously installed
+dependency, an error will be raised.
+
+Ensure that ``constrain_package_deps = true`` is set in the test environment in order to use the constraints file
+generated by processing the ``deps`` section when performing ``package_deps``.
 
 Note constraint files are a subset of requirement files. Therefore, it's valid to pass a constraint file wherever you
 can specify a requirement file.

src/tox/config/loader/ini/replace.py

@@ -30,7 +30,7 @@
 ARG_DELIMITER = ":"
 REPLACE_START = "{"
 REPLACE_END = "}"
-BACKSLASH_ESCAPE_CHARS = ["\\", ARG_DELIMITER, REPLACE_START, REPLACE_END, "[", "]"]
+BACKSLASH_ESCAPE_CHARS = [ARG_DELIMITER, REPLACE_START, REPLACE_END, "[", "]"]
 MAX_REPLACE_DEPTH = 100
 
 
@@ -115,11 +115,18 @@ def parse_and_split_to_terminator(
         pos = 0
 
         while pos < len(value):
-            if len(value) > pos + 1 and value[pos] == "\\" and value[pos + 1] in BACKSLASH_ESCAPE_CHARS:
-                # backslash escapes the next character from a special set
-                last_arg.append(value[pos + 1])
-                pos += 2
-                continue
+            if len(value) > pos + 1 and value[pos] == "\\":
+                if value[pos + 1] in BACKSLASH_ESCAPE_CHARS:
+                    # backslash escapes the next character from a special set
+                    last_arg.append(value[pos + 1])
+                    pos += 2
+                    continue
+                if value[pos + 1] == "\\":
+                    # backlash doesn't escape a backslash, but does prevent it from affecting the next char
+                    # a subsequent `shlex` pass will eat the double backslash during command splitting
+                    last_arg.append(value[pos : pos + 2])
+                    pos += 2
+                    continue
             fragment = value[pos:]
             if terminator and fragment.startswith(terminator):
                 pos += len(terminator)

src/tox/config/loader/str_convert.py

@@ -45,11 +45,43 @@ def to_dict(value: str, of_type: tuple[type[Any], type[Any]]) -> Iterator[tuple[
                 else:
                     raise TypeError(f"dictionary lines must be of form key=value, found {row!r}")
 
+    @staticmethod
+    def _win32_process_path_backslash(value: str, escape: str, special_chars: str) -> str:
+        """
+        Escape backslash in value that is not followed by a special character.
+
+        This allows windows paths to be written without double backslash, while
+        retaining the POSIX backslash escape semantics for quotes and escapes.
+        """
+        result = []
+        for ix, char in enumerate(value):
+            result.append(char)
+            if char == escape:
+                last_char = value[ix - 1 : ix]
+                if last_char == escape:
+                    continue
+                next_char = value[ix + 1 : ix + 2]
+                if next_char not in (escape, *special_chars):
+                    result.append(escape)  # escape escapes that are not themselves escaping a special character
+        return "".join(result)
+
     @staticmethod
     def to_command(value: str) -> Command:
-        is_win = sys.platform == "win32"
+        """
+        At this point, ``value`` has already been substituted out, and all punctuation / escapes are final.
+
+        Value will typically be stripped of whitespace when coming from an ini file.
+        """
         value = value.replace(r"\#", "#")
-        splitter = shlex.shlex(value, posix=not is_win)
+        is_win = sys.platform == "win32"
+        if is_win:  # pragma: win32 cover
+            s = shlex.shlex(posix=True)
+            value = StrConvert._win32_process_path_backslash(
+                value,
+                escape=s.escape,
+                special_chars=s.quotes + s.whitespace,
+            )
+        splitter = shlex.shlex(value, posix=True)
         splitter.whitespace_split = True
         splitter.commenters = ""  # comments handled earlier, and the shlex does not know escaped comment characters
         args: list[str] = []

src/tox/pytest.py

@@ -525,6 +525,7 @@ def register_inline_plugin(mocker: MockerFixture, *args: Callable[..., Any]) ->
     "LogCaptureFixture",
     "TempPathFactory",
     "MonkeyPatch",
+    "SubRequest",
     "ToxRunOutcome",
     "ToxProject",
     "ToxProjectCreator",

src/tox/tox_env/python/pip/pip_install.py

@@ -2,6 +2,7 @@
 
 import logging
 from collections import defaultdict
+from pathlib import Path
 from typing import Any, Callable, Sequence
 
 from packaging.requirements import Requirement
@@ -38,6 +39,18 @@ def _register_config(self) -> None:
             post_process=self.post_process_install_command,
             desc="command used to install packages",
         )
+        self._env.conf.add_config(
+            keys=["constrain_package_deps"],
+            of_type=bool,
+            default=True,
+            desc="If true, apply constraints during install_package_deps.",
+        )
+        self._env.conf.add_config(
+            keys=["use_frozen_constraints"],
+            of_type=bool,
+            default=False,
+            desc="Use the exact versions of installed deps as constraints, otherwise use the listed deps.",
+        )
         if self._with_list_deps:  # pragma: no branch
             self._env.conf.add_config(
                 keys=["list_dependencies_command"],
@@ -81,6 +94,17 @@ def install(self, arguments: Any, section: str, of_type: str) -> None:
             logging.warning(f"pip cannot install {arguments!r}")
             raise SystemExit(1)
 
+    def constraints_file(self) -> Path:
+        return Path(self._env.env_dir) / "constraints.txt"
+
+    @property
+    def constrain_package_deps(self) -> bool:
+        return bool(self._env.conf["constrain_package_deps"])
+
+    @property
+    def use_frozen_constraints(self) -> bool:
+        return bool(self._env.conf["use_frozen_constraints"])
+
     def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type: str) -> None:
         try:
             new_options, new_reqs = arguments.unroll()
@@ -90,7 +114,16 @@ def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type
         new_constraints: list[str] = []
         for req in new_reqs:
             (new_constraints if req.startswith("-c ") else new_requirements).append(req)
-        new = {"options": new_options, "requirements": new_requirements, "constraints": new_constraints}
+        constraint_options = {
+            "constrain_package_deps": self.constrain_package_deps,
+            "use_frozen_constraints": self.use_frozen_constraints,
+        }
+        new = {
+            "options": new_options,
+            "requirements": new_requirements,
+            "constraints": new_constraints,
+            "constraint_options": constraint_options,
+        }
         # if option or constraint change in any way recreate, if the requirements change only if some are removed
         with self._env.cache.compare(new, section, of_type) as (eq, old):
             if not eq:  # pragma: no branch
@@ -100,9 +133,16 @@ def _install_requirement_file(self, arguments: PythonDeps, section: str, of_type
                     missing_requirement = set(old["requirements"]) - set(new_requirements)
                     if missing_requirement:
                         raise Recreate(f"requirements removed: {' '.join(missing_requirement)}")
+                    old_constraint_options = old.get("constraint_options")
+                    if old_constraint_options != constraint_options:
+                        msg = f"constraint options changed: old={old_constraint_options} new={constraint_options}"
+                        raise Recreate(msg)
                 args = arguments.as_root_args
                 if args:  # pragma: no branch
                     self._execute_installer(args, of_type)
+                    if self.constrain_package_deps and not self.use_frozen_constraints:
+                        combined_constraints = new_requirements + [c.lstrip("-c ") for c in new_constraints]
+                        self.constraints_file().write_text("\n".join(combined_constraints))
 
     @staticmethod
     def _recreate_if_diff(of_type: str, new_opts: list[str], old_opts: list[str], fmt: Callable[[str], str]) -> None:
@@ -155,10 +195,19 @@ def _install_list_of_deps(
             self._execute_installer(install_args, of_type)
 
     def _execute_installer(self, deps: Sequence[Any], of_type: str) -> None:
+        if of_type == "package_deps" and self.constrain_package_deps:
+            constraints_file = self.constraints_file()
+            if constraints_file.exists():
+                deps = [*deps, f"-c{constraints_file}"]
+
         cmd = self.build_install_cmd(deps)
         outcome = self._env.execute(cmd, stdin=StdinSource.OFF, run_id=f"install_{of_type}")
         outcome.assert_success()
 
+        if of_type == "deps" and self.constrain_package_deps and self.use_frozen_constraints:
+            # freeze installed deps for use as constraints
+            self.constraints_file().write_text("\n".join(self.installed()))
+
     def build_install_cmd(self, args: Sequence[str]) -> list[str]:
         try:
             cmd: Command = self._env.conf["install_command"]

tests/config/loader/ini/replace/test_replace_env_var.py

@@ -17,24 +17,24 @@ def test_replace_env_set(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> N
 
 
 def test_replace_env_set_double_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
-    """Double backslash should escape to single backslash and not affect surrounding replacements."""
+    """Double backslash should remain but not affect surrounding replacements."""
     monkeypatch.setenv("MAGIC", "something good")
     result = replace_one(r"{env:MAGIC}\\{env:MAGIC}")
-    assert result == r"something good\something good"
+    assert result == r"something good\\something good"
 
 
 def test_replace_env_set_triple_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
-    """Triple backslash should escape to single backslash also escape subsequent replacement."""
+    """Triple backslash should retain two slashes with the third escaping subsequent replacement."""
     monkeypatch.setenv("MAGIC", "something good")
     result = replace_one(r"{env:MAGIC}\\\{env:MAGIC}")
-    assert result == r"something good\{env:MAGIC}"
+    assert result == r"something good\\{env:MAGIC}"
 
 
 def test_replace_env_set_quad_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
-    """Quad backslash should escape to two backslashes and not affect surrounding replacements."""
+    """Quad backslash should remain but not affect surrounding replacements."""
     monkeypatch.setenv("MAGIC", "something good")
-    result = replace_one(r"\\{env:MAGIC}\\\\{env:MAGIC}\\")
-    assert result == r"\something good\\something good" + "\\"
+    result = replace_one(r"\\{env:MAGIC}\\\\{env:MAGIC}" + "\\")
+    assert result == r"\\something good\\\\something good" + "\\"
 
 
 def test_replace_env_when_value_is_backslash(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None: