docs/changelog.rst

@@ -4,6 +4,28 @@ Release History
 
 .. towncrier release notes start
 
+v4.3.0 (2023-01-15)
+-------------------
+
+Features - 4.3.0
+~~~~~~~~~~~~~~~~
+- Rewrite substitution replacement parser - by :user:`masenf`
+
+  * ``\`` acts as a proper escape for ``\`` in ini-style substitutions
+  * The resulting value of a substitution is no longer reprocessed in the context
+    of the broader string. (Prior to this change, ini-values were repeatedly re-substituted until
+    the expression no longer had modifications)
+  * Migrate and update "Substitutions" section of Configuration page from v3 docs.
+  * ```find_replace_part`` is removed from ``tox.config.loader.ini.replace``
+  * New names exported from ``tox.config.loader.ini.replace``:
+      * ``find_replace_expr``
+      * ``MatchArg``
+      * ``MatchError``
+      * ``MatchExpression``
+      * Note: the API for ``replace`` itself is unchanged. (:issue:`2732`)
+- Improved documentation for factors and test env names - by :user:`stephenfin`. (:issue:`2852`)
+
+
 v4.2.8 (2023-01-11)
 -------------------
 

docs/config.rst

@@ -97,6 +97,8 @@ examples first and use this page as a reference.
         commands = mypy src
     """
 
+.. _conf-core:
+
 Core
 ----
 
@@ -230,7 +232,7 @@ Python language core options
     :ref:`base_python` and instead always use the base Python implied from the Python name. This allows you to configure
     :ref:`base_python` in the :ref:`base` section without affecting environments that have implied base Python versions.
 
-
+.. _conf-testenv:
 
 tox environment
 ---------------
@@ -763,3 +765,144 @@ Example configuration:
 
     [tox]
     skip_missing_interpreters = true
+
+Substitutions
+-------------
+
+Any ``key=value`` setting in an ini-file can make use of **value substitution**
+through the ``{...}`` string-substitution pattern.
+
+The string inside the curly braces may reference a global or per-environment config key as described above.
+
+The backslash character ``\`` will act as an escape for a following: ``\``,
+``{``, ``}``, ``:``, ``[``, or ``]``, otherwise the backslash will be
+reproduced literally::
+
+    commands =
+        python -c 'print("\{posargs} = \{}".format("{posargs}"))'
+        python -c 'print("host: \{}".format("{env:HOSTNAME:host\: not set}")'
+
+Special substitutions that accept additional colon-delimited ``:`` parameters
+cannot have a space after the ``:`` at the beginning of line (e.g.  ``{posargs:
+magic}`` would be parsed as factorial ``{posargs``, having value magic).
+
+Environment variable substitutions
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you specify a substitution string like this::
+
+    {env:KEY}
+
+then the value will be retrieved as ``os.environ['KEY']``
+and raise an Error if the environment variable
+does not exist.
+
+
+Environment variable substitutions with default values
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+If you specify a substitution string like this::
+
+    {env:KEY:DEFAULTVALUE}
+
+then the value will be retrieved as ``os.environ['KEY']``
+and replace with DEFAULTVALUE if the environment variable does not
+exist.
+
+If you specify a substitution string like this::
+
+    {env:KEY:}
+
+then the value will be retrieved as ``os.environ['KEY']``
+and replace with an empty string if the environment variable does not
+exist.
+
+Substitutions can also be nested. In that case they are expanded starting
+from the innermost expression::
+
+    {env:KEY:{env:DEFAULT_OF_KEY}}
+
+the above example is roughly equivalent to
+``os.environ.get('KEY', os.environ['DEFAULT_OF_KEY'])``
+
+Interactive shell substitution
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 3.4.0
+
+It's possible to inject a config value only when tox is running in interactive shell (standard input)::
+
+    {tty:ON_VALUE:OFF_VALUE}
+
+The first value is the value to inject when the interactive terminal is
+available, the second value is the value to use when it's not (optiona). A good
+use case for this is e.g. passing in the ``--pdb`` flag for pytest.
+
+.. _`command positional substitution`:
+.. _`positional substitution`:
+
+Substitutions for positional arguments in commands
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.0
+
+If you specify a substitution string like this::
+
+    {posargs:DEFAULTS}
+
+then the value will be replaced with positional arguments as provided
+to the tox command::
+
+    tox arg1 arg2
+
+In this instance, the positional argument portion will be replaced with
+``arg1 arg2``. If no positional arguments were specified, the value of
+DEFAULTS will be used instead. If DEFAULTS contains other substitution
+strings, such as ``{env:*}``, they will be interpreted.,
+
+Use a double ``--`` if you also want to pass options to an underlying
+test command, for example::
+
+    tox -- --opt1 ARG1
+
+will make the ``--opt1 ARG1`` appear in all test commands where ``[]`` or
+``{posargs}`` was specified.  By default (see ``args_are_paths``
+setting), ``tox`` rewrites each positional argument if it is a relative
+path and exists on the filesystem to become a path relative to the
+``changedir`` setting.
+
+Substitution for values from other sections
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+
+.. versionadded:: 1.4
+
+Values from other sections can be referred to via::
+
+   {[sectionname]valuename}
+
+which you can use to avoid repetition of config values.
+You can put default values in one section and reference them in others to avoid repeating the same values:
+
+.. code-block:: ini
+
+    [base]
+    deps =
+        pytest
+        mock
+        pytest-xdist
+
+    [testenv:dulwich]
+    deps =
+        dulwich
+        {[base]deps}
+
+    [testenv:mercurial]
+    deps =
+        mercurial
+        {[base]deps}
+
+Other Substitutions
+~~~~~~~~~~~~~~~~~~~
+
+* ``{}`` - replaced as ``os.pathsep``
+* ``{/}`` - replaced as ``os.sep``

docs/user_guide.rst

@@ -1,32 +1,135 @@
 User Guide
 ==========
 
-Basic example
--------------
+Overview
+--------
 
 tox is an environment orchestrator. Use it to define how to setup and execute various tools on your projects. The
-tool can be:
+tool can set up environments for and invoke:
+
+- test runners (such as :pypi:`pytest`),
+- linters (e.g., :pypi:`flake8`),
+- formatters (for example :pypi:`black` or :pypi:`isort`),
+- documentation generators (e.g., :pypi:`Sphinx`),
+- build and publishing tools (e.g., :pypi:`build` with :pypi:`twine`),
+- ...
+
+Configuration
+-------------
+
+*tox* needs a configuration file where you define what tools you need to run and how to provision a test environment for
+these. The canonical file for this is the ``tox.ini`` file. For example:
+
+.. code-block:: ini
+
+    [tox]
+    requires =
+        tox>=4
+    env_list = lint, type, py{38,39,310,311}
+
+    [testenv]
+    description = run unit tests
+    deps =
+        pytest>=7
+        pytest-sugar
+    commands =
+        pytest {posargs:tests}
 
-- a test runner (such as :pypi:`pytest`),
-- a linter (e.g., :pypi:`flake8`),
-- a formatter (for example :pypi:`black` or :pypi:`isort`),
-- a documentation generator (e.g., :pypi:`Sphinx`),
-- library builder and publisher (e.g., :pypi:`build` with :pypi:`twine`),
-- or anything else you may need to execute.
+    [testenv:lint]
+    description = run linters
+    skip_install = true
+    deps =
+        black==22.12
+    commands = black {posargs:.}
 
-First, in a configuration file you need to define what tools you need to run and how to provision a test environment for
-these. The canonical file for this is the ``tox.ini`` file, let's take a look at an example of this (this needs to live
-at the root of your project):
+    [testenv:type]
+    description = run type checks
+    deps =
+        mypy>=0.991
+    commands =
+        mypy {posargs:src tests}
 
-.. note::
+.. tip::
 
    You can also generate a ``tox.ini`` file automatically by running ``tox quickstart`` and then answering a few
    questions.
 
+The configuration is split into two type of configuration: core settings are hosted under a core ``tox`` section while
+per run environment settings hosted under ``testenv`` and ``testenv:<env_name>`` sections.
+
+Core settings
+~~~~~~~~~~~~~
+
+Core settings that affect all test environments or configure how tox itself is invoked are defined under the ``tox``
+section.
+
 .. code-block:: ini
 
     [tox]
-    envlist =
+    requires =
+        tox>=4
+    env_list = lint, type, py{38,39,310,311}
+
+We can use it to specify things such as the minimum version of *tox* required or the location of the package under test.
+A list of all supported configuration options for the ``tox`` section can be found in the :ref:`configuration guide
+<conf-core>`.
+
+Test environments
+~~~~~~~~~~~~~~~~~
+
+Test environments are defined under the ``testenv`` section and individual ``testenv:<env_name>`` sections, where
+``<env_name>`` is the name of a specific environment.
+
+.. code-block:: ini
+
+    [testenv]
+    description = run unit tests
+    deps =
+        pytest>=7
+        pytest-sugar
+    commands =
+        pytest {posargs:tests}
+
+    [testenv:lint]
+    description = run linters
+    skip_install = true
+    deps =
+        black==22.12
+    commands = black {posargs:.}
+
+    [testenv:type]
+    description = run type checks
+    deps =
+        mypy>=0.991
+    commands =
+        mypy {posargs:src tests}
+
+Settings defined in the top-level ``testenv`` section are automatically inherited by individual environments unless
+overridden. Test environment names can consist of alphanumeric characters and dashes; for example: ``py311-django42``.
+The name will be split on dashes into multiple factors, meaning ``py311-django42`` will be split into two factors:
+``py311`` and ``django42``. *tox* defines a number of default factors, which correspond to various versions and
+implementations of Python and provide default values for ``base_python``:
+
+- ``pyNM``: configures ``basepython = pythonN.M``
+- ``pypyNM``: configures ``basepython = pypyN.M``
+- ``jythonNM``: configures ``basepython = jythonN.M``
+- ``cpythonNM``: configures ``basepython = cpythonN.M``
+- ``ironpythonNM``: configures ``basepython = ironpythonN.M``
+- ``rustpythonNM``: configures ``basepython = rustpythonN.M``
+
+You can also specify these factors with a period between the major and minor versions (e.g. ``pyN.M``), without a minor
+version (e.g. ``pyN``), or without any version information whatsoever (e.g. ``py``)
+
+A list of all supported configuration options for the ``testenv`` and ``testenv:<env_name>`` sections can be found in
+the :ref:`configuration guide <conf-testenv>`.
+
+Basic example
+-------------
+
+.. code-block:: ini
+
+    [tox]
+    env_list =
         format
         py310
 
@@ -43,25 +146,25 @@ at the root of your project):
         pytest-sugar
     commands = pytest tests {posargs}
 
+This example contains a global ``tox`` section as well as two test environments. Taking the core section first, we use
+the :ref:`env_list` setting to indicate that this project has two run environments named ``format`` and ``py310`` that
+should be run by default when ``tox run`` is invoked without a specific environment.
 
-The configuration is split into two type of configuration: core settings are hosted under the ``tox`` section and per run
-environment settings hosted under ``testenv:<env_name>``. Under the core section we define that this project has two
-run environments named ``format`` and ``py310`` respectively (we use the ``envlist`` configuration key to do so).
+The formatting environment and test environment are defined separately via the ``testenv:format`` and ``testenv:py310``
+sections, respectively. For example to format the project we:
 
-Then we define separately the formatting environment (``testenv:format`` section) and the test environment
-(``testenv:py310`` section). For example to format the project we:
+- add a description (visible when you type ``tox list`` into the command line) via the :ref:`description` setting
+- define that it requires the :pypi:`black` dependency with version ``22.3.0`` via the :ref:`deps` setting
+- disable installation of the project under test into the test environment via the :ref:`skip_install` setting -
+  ``black`` does not need it installed
+- indicate the commands to be run via the :ref:`commands` setting
 
-- add a description (visible when you type ``tox list`` into the command line),
-- we define that it requires the ``black`` PyPI dependency with version ``22.3.0``,
-- the black tool does not need the project we are testing to be installed into  the test environment so we disable this
-  default behaviour via the ``skip_install`` configuration,
-- and we define that the tool should be invoked as we'd type ``black .`` into the command line.
+For testing the project we use the ``py310`` environment. For this environment we:
 
-For testing the project we use the ``py310`` environment, for which we:
-
-- define a text description of the environment,
-- specify that requires ``pytest`` ``7`` or later together with the :pypi:`pytest-sugar` project,
-- and that the tool should be invoked via the ``pytest tests`` CLI command.
+- define a text description of the environment via the :ref:`description` setting
+- specify that we should install :pypi:`pytest` v7.0 or later together with the :pypi:`pytest-sugar` project via the
+  :ref:`deps` setting
+- indicate the command(s) to be run - in this case ``pytest tests`` - via the :ref:`commands` setting
 
 ``{posargs}`` is a place holder part for the CLI command that allows us to pass additional flags to the pytest
 invocation, for example if we'd want to run ``pytest tests -v`` as a one off, instead of ``tox run -e py310`` we'd type
@@ -232,8 +335,8 @@ CLI
   have a stale Python environment; e.g. ``tox run -e py310 -r`` would clean the run environment and recreate it from
   scratch.
 
-Configuration
-~~~~~~~~~~~~~
+Config files
+~~~~~~~~~~~~
 
 - Every tox environment has its own configuration section (e.g. in case of ``tox.ini`` configuration method the
   ``py310`` tox environments configuration is read from the ``testenv:py310`` section). If the section is missing or does
@@ -259,7 +362,9 @@ Configuration
   - To view environment variables set and passed down use ``tox c -e py310 -k set_env pass_env``.
   - To pass through additional environment variables use :ref:`pass_env`.
   - To set environment variables use :ref:`set_env`.
+
 - Setup operation can be configured via the :ref:`commands_pre`, while teardown commands via the :ref:`commands_post`.
+
 - Configurations may be set conditionally within the ``tox.ini`` file. If a line starts with an environment name
   or names, separated by a comma, followed by ``:`` the configuration will only be used if the
   environment name(s) matches the executed tox environment. For example:
@@ -279,6 +384,7 @@ Configuration
 
 Parallel mode
 -------------
+
 ``tox`` allows running environments in parallel mode via the ``parallel`` sub-command:
 
 - After the packaging phase completes tox will run the tox environments in parallel processes (multi-thread based).

pyproject.toml

@@ -10,6 +10,7 @@ readme.content-type = "text/markdown"
 keywords = ["virtual", "environments", "isolated", "testing"]
 license = "MIT"
 urls.Homepage = "http://tox.readthedocs.org"
+urls.Documentation = "https://tox.wiki"
 urls.Source = "https://github.com/tox-dev/tox"
 urls.Tracker = "https://github.com/tox-dev/tox/issues"
 urls."Release Notes" = "https://tox.wiki/en/latest/changelog.html"

src/tox/config/loader/ini/replace.py

@@ -9,7 +9,7 @@
 from configparser import SectionProxy
 from functools import lru_cache
 from pathlib import Path
-from typing import TYPE_CHECKING, Iterator, Pattern
+from typing import TYPE_CHECKING, Any, Iterator, Pattern, Sequence, Union
 
 from tox.config.loader.api import ConfigLoadArgs
 from tox.config.loader.stringify import stringify
@@ -21,74 +21,175 @@
     from tox.config.loader.ini import IniLoader
     from tox.config.main import Config
 
-# split alongside :, unless it's escaped, or it's preceded by a single capital letter (Windows drive letter in paths)
-ARGS_GROUP = re.compile(r"(?<!\\\\|:[A-Z]):")
+# split alongside :, unless it's preceded by a single capital letter (Windows drive letter in paths)
+ARG_DELIMITER = ":"
+REPLACE_START = "{"
+REPLACE_END = "}"
+BACKSLASH_ESCAPE_CHARS = ["\\", ARG_DELIMITER, REPLACE_START, REPLACE_END, "[", "]"]
+
+
+MatchArg = Sequence[Union[str, "MatchExpression"]]
+
+
+def find_replace_expr(value: str) -> MatchArg:
+    """Find all replaceable tokens within value."""
+    return MatchExpression.parse_and_split_to_terminator(value)[0][0]
 
 
 def replace(conf: Config, loader: IniLoader, value: str, args: ConfigLoadArgs) -> str:
-    # perform all non-escaped replaces
-    end = 0
-    while True:
-        start, end, to_replace = find_replace_part(value, end)
-        if to_replace is None:
-            break
-        replaced = _replace_match(conf, loader, to_replace, args.copy())
-        if replaced is None:
-            # if we cannot replace, keep what was there, and continue looking for additional replaces following
-            # note, here we cannot raise because the content may be a factorial expression, and in those case we don't
-            # want to enforce escaping curly braces, e.g. it should work to write: env_list = {py39,py38}-{,dep}
-            end = end + 1
-            continue
-        new_value = f"{value[:start]}{replaced}{value[end + 1:]}"
-        end = 0  # if we performed a replacement start over
-        if new_value == value:  # if we're not making progress stop (circular reference?)
-            break
-        value = new_value
-    # remove escape sequences
-    value = value.replace("\\{", "{")
-    value = value.replace("\\}", "}")
-    value = value.replace("\\[", "[")
-    value = value.replace("\\]", "]")
-    return value
-
-
-REPLACE_PART = re.compile(
-    r"""
-        (?<!\\) {  # Unescaped {
-            ( [^{},] | \\ { | \\ } )*  # Anything except an unescaped { or }
-        (?<! \\) }  # Unescaped }
-    |
-        (?<! \\) \[ ]  # Unescaped []
-    """,
-    re.VERBOSE,
-)
+    """Replace all active tokens within value according to the config."""
+    return Replacer(conf, loader, conf_args=args).join(find_replace_expr(value))
 
 
-def find_replace_part(value: str, end: int) -> tuple[int, int, str | None]:
-    match = REPLACE_PART.search(value, end)
-    if match is None:
-        return -1, -1, None
-    if match.group() == "[]":
-        return match.start(), match.end() - 1, "posargs"  # brackets is an alias for positional arguments
-    matched_part = match.group()[1:-1]
-    return match.start(), match.end() - 1, matched_part
-
-
-def _replace_match(conf: Config, loader: IniLoader, value: str, conf_args: ConfigLoadArgs) -> str | None:
-    of_type, *args = ARGS_GROUP.split(value)
-    if of_type == "/":
-        replace_value: str | None = os.sep
-    elif of_type == "" and args == [""]:
-        replace_value = os.pathsep
-    elif of_type == "env":
-        replace_value = replace_env(conf, args, conf_args)
-    elif of_type == "tty":
-        replace_value = replace_tty(args)
-    elif of_type == "posargs":
-        replace_value = replace_pos_args(conf, args, conf_args)
-    else:
-        replace_value = replace_reference(conf, loader, value, conf_args)
-    return replace_value
+class MatchError(Exception):
+    """Could not find end terminator in MatchExpression."""
+
+
+class MatchExpression:
+    """An expression that is handled specially by the Replacer."""
+
+    def __init__(self, expr: Sequence[MatchArg], term_pos: int | None = None):
+        self.expr = expr
+        self.term_pos = term_pos
+
+    def __repr__(self) -> str:
+        return f"MatchExpression(expr={self.expr!r}, term_pos={self.term_pos!r})"
+
+    def __eq__(self, other: Any) -> bool:
+        if isinstance(other, type(self)):
+            return self.expr == other.expr
+        return NotImplemented
+
+    @classmethod
+    def _next_replace_expression(cls, value: str) -> MatchExpression | None:
+        """Process a curly brace replacement expression."""
+        if value.startswith("[]"):
+            # `[]` is shorthand for `{posargs}`
+            return MatchExpression(expr=[["posargs"]], term_pos=1)
+        if not value.startswith(REPLACE_START):
+            return None
+        try:
+            # recursively handle inner expression
+            rec_expr, term_pos = cls.parse_and_split_to_terminator(
+                value[1:],
+                terminator=REPLACE_END,
+                split=ARG_DELIMITER,
+            )
+        except MatchError:
+            # did NOT find the expected terminator character, so treat `{` as if escaped
+            pass
+        else:
+            return MatchExpression(expr=rec_expr, term_pos=term_pos)
+        return None
+
+    @classmethod
+    def parse_and_split_to_terminator(
+        cls,
+        value: str,
+        terminator: str = "",
+        split: str | None = None,
+    ) -> tuple[Sequence[MatchArg], int]:
+        """
+        Tokenize `value` to up `terminator` character.
+
+        If `split` is given, multiple arguments will be returned.
+
+        Returns list of arguments (list of str or MatchExpression) and final character position examined in value.
+
+        This function recursively calls itself via `_next_replace_expression`.
+        """
+        args = []
+        last_arg: list[str | MatchExpression] = []
+        pos = 0
+
+        while pos < len(value):
+            if len(value) > pos + 1 and value[pos] == "\\" and value[pos + 1] in BACKSLASH_ESCAPE_CHARS:
+                # backslash escapes the next character from a special set
+                last_arg.append(value[pos + 1])
+                pos += 2
+                continue
+            fragment = value[pos:]
+            if terminator and fragment.startswith(terminator):
+                pos += len(terminator)
+                break
+            if split and fragment.startswith(split):
+                # found a new argument
+                args.append(last_arg)
+                last_arg = []
+                pos += len(split)
+                continue
+            expr = cls._next_replace_expression(fragment)
+            if expr is not None:
+                pos += (expr.term_pos or 0) + 1
+                last_arg.append(expr)
+                continue
+            # default case: consume the next character
+            last_arg.append(value[pos])
+            pos += 1
+        else:  # fell out of the loop
+            if terminator:
+                raise MatchError(f"{terminator!r} remains unmatched in {value!r}")
+        args.append(last_arg)
+        return [_flatten_string_fragments(a) for a in args], pos
+
+
+def _flatten_string_fragments(seq_of_str_or_other: Sequence[str | Any]) -> Sequence[str | Any]:
+    """Join runs of contiguous str values in a sequence; nny non-str items in the sequence are left as-is."""
+    result = []
+    last_str = []
+    for obj in seq_of_str_or_other:
+        if isinstance(obj, str):
+            last_str.append(obj)
+        else:
+            if last_str:
+                result.append("".join(last_str))
+                last_str = []
+            result.append(obj)
+    if last_str:
+        result.append("".join(last_str))
+    return result
+
+
+class Replacer:
+    """Recursively expand MatchExpression against the config and loader."""
+
+    def __init__(self, conf: Config, loader: IniLoader, conf_args: ConfigLoadArgs):
+        self.conf = conf
+        self.loader = loader
+        self.conf_args = conf_args
+
+    def __call__(self, value: MatchArg) -> Sequence[str]:
+        return [self._replace_match(me) if isinstance(me, MatchExpression) else str(me) for me in value]
+
+    def join(self, value: MatchArg) -> str:
+        return "".join(self(value))
+
+    def _replace_match(self, value: MatchExpression) -> str:
+        of_type, *args = flattened_args = [self.join(arg) for arg in value.expr]
+        if of_type == "/":
+            replace_value: str | None = os.sep
+        elif of_type == "" and args == [""]:
+            replace_value = os.pathsep
+        elif of_type == "env":
+            replace_value = replace_env(self.conf, args, self.conf_args)
+        elif of_type == "tty":
+            replace_value = replace_tty(args)
+        elif of_type == "posargs":
+            replace_value = replace_pos_args(self.conf, args, self.conf_args)
+        else:
+            replace_value = replace_reference(
+                self.conf,
+                self.loader,
+                ARG_DELIMITER.join(flattened_args),
+                self.conf_args,
+            )
+        if replace_value is not None:
+            return replace_value
+        # else: fall through -- when replacement is not possible, treat `{` as if escaped.
+        #     If we cannot replace, keep what was there, and continue looking for additional replaces
+        #     NOTE: cannot raise because the content may be a factorial expression where we don't
+        #           want to enforce escaping curly braces, e.g. `env_list = {py39,py38}-{,dep}` should work
+        return f"{REPLACE_START}%s{REPLACE_END}" % ARG_DELIMITER.join(flattened_args)
 
 
 @lru_cache(maxsize=None)
@@ -98,6 +199,7 @@ def _replace_ref(env: str | None) -> Pattern[str]:
     (\[(?P<full_env>{re.escape(env or '.*')}(:(?P<env>[^]]+))?|(?P<section>[-\w]+))])? # env/section
     (?P<key>[-a-zA-Z0-9_]+) # key
     (:(?P<default>.*))? # default value
+    $
 """,
         re.VERBOSE,
     )
@@ -179,13 +281,15 @@ def replace_pos_args(conf: Config, args: list[str], conf_args: ConfigLoadArgs) -
             pass
     pos_args = conf.pos_args(to_path)
     if pos_args is None:
-        replace_value = ":".join(args)  # if we use the defaults join back remaining args
+        replace_value = ARG_DELIMITER.join(args)  # if we use the defaults join back remaining args
     else:
         replace_value = shell_cmd(pos_args)
     return replace_value
 
 
 def replace_env(conf: Config, args: list[str], conf_args: ConfigLoadArgs) -> str:
+    if not args or not args[0]:
+        raise MatchError("No variable name was supplied in {env} substitution")
     key = args[0]
     new_key = f"env:{key}"
 
@@ -203,7 +307,7 @@ def replace_env(conf: Config, args: list[str], conf_args: ConfigLoadArgs) -> str
     if key in os.environ:
         return os.environ[key]
 
-    return "" if len(args) == 1 else ":".join(args[1:])
+    return "" if len(args) == 1 else ARG_DELIMITER.join(args[1:])
 
 
 def replace_tty(args: list[str]) -> str:
@@ -215,6 +319,9 @@ def replace_tty(args: list[str]) -> str:
 
 
 __all__ = (
+    "find_replace_expr",
+    "MatchArg",
+    "MatchError",
+    "MatchExpression",
     "replace",
-    "find_replace_part",
 )

src/tox/config/set_env.py

@@ -18,7 +18,7 @@ def __init__(self, raw: str, name: str, env_name: str | None, root: Path) -> Non
         self._env_files: list[str] = []
         self._replacer: Replacer = lambda s, c: s  # noqa: U100
         self._name, self._env_name, self._root = name, env_name, root
-        from .loader.ini.replace import find_replace_part
+        from .loader.ini.replace import MatchExpression, find_replace_expr
 
         for line in raw.splitlines():
             if line.strip():
@@ -30,9 +30,10 @@ def __init__(self, raw: str, name: str, env_name: str | None, root: Path) -> Non
                         if "{" in key:
                             raise ValueError(f"invalid line {line!r} in set_env")
                     except ValueError:
-                        _, __, match = find_replace_part(line, 0)
-                        if match:
-                            self._needs_replacement.append(line)
+                        for expr in find_replace_expr(line):
+                            if isinstance(expr, MatchExpression):
+                                self._needs_replacement.append(line)
+                                break
                         else:
                             raise
                     else:

tests/config/loader/ini/replace/test_replace.py

@@ -2,26 +2,92 @@
 
 import pytest
 
-from tox.config.loader.ini.replace import find_replace_part
+from tests.config.loader.ini.replace.conftest import ReplaceOne
+from tox.config.loader.ini.replace import MatchExpression, find_replace_expr
+from tox.report import HandledError
 
 
 @pytest.mark.parametrize(
-    ("value", "result"),
+    ("value", "exp_output"),
     [
-        ("[]", (0, 1, "posargs")),
-        ("123[]", (3, 4, "posargs")),
-        ("[]123", (0, 1, "posargs")),
-        (r"\[\] []", (5, 6, "posargs")),
-        (r"[\] []", (4, 5, "posargs")),
-        (r"\[] []", (4, 5, "posargs")),
-        ("{foo}", (0, 4, "foo")),
-        (r"\{foo} {bar}", (7, 11, "bar")),
-        ("{foo} {bar}", (0, 4, "foo")),
-        (r"{foo\} {bar}", (7, 11, "bar")),
-        (r"{foo:{bar}}", (5, 9, "bar")),
-        (r"{\{}", (0, 3, r"\{")),
-        (r"{\}}", (0, 3, r"\}")),
+        ("[]", [MatchExpression([["posargs"]])]),
+        ("123[]", ["123", MatchExpression([["posargs"]])]),
+        ("[]123", [MatchExpression([["posargs"]]), "123"]),
+        (r"\[\] []", ["[] ", MatchExpression([["posargs"]])]),
+        (r"[\] []", ["[] ", MatchExpression([["posargs"]])]),
+        (r"\[] []", ["[] ", MatchExpression([["posargs"]])]),
+        ("{foo}", [MatchExpression([["foo"]])]),
+        (r"\{foo} {bar}", ["{foo} ", MatchExpression([["bar"]])]),
+        ("{foo} {bar}", [MatchExpression([["foo"]]), " ", MatchExpression([["bar"]])]),
+        (r"{foo\} {bar}", ["{foo} ", MatchExpression([["bar"]])]),
+        (r"{foo:{bar}}", [MatchExpression([["foo"], [MatchExpression([["bar"]])]])]),
+        (r"{foo\::{bar}}", [MatchExpression([["foo:"], [MatchExpression([["bar"]])]])]),
+        (r"{foo:B:c:D:e}", [MatchExpression([["foo"], ["B"], ["c"], ["D"], ["e"]])]),
+        (r"{\{}", [MatchExpression([["{"]])]),
+        (r"{\}}", [MatchExpression([["}"]])]),
+        (
+            r"p{foo:b{a{r}:t}:{ba}z}s",
+            [
+                "p",
+                MatchExpression(
+                    [
+                        ["foo"],
+                        [
+                            "b",
+                            MatchExpression(
+                                [
+                                    ["a", MatchExpression([["r"]])],
+                                    ["t"],
+                                ],
+                            ),
+                        ],
+                        [
+                            MatchExpression(
+                                [["ba"]],
+                            ),
+                            "z",
+                        ],
+                    ],
+                ),
+                "s",
+            ],
+        ),
+        ("\\", ["\\"]),
+        (r"\d", ["\\d"]),
+        (r"C:\WINDOWS\foo\bar", [r"C:\WINDOWS\foo\bar"]),
     ],
 )
-def test_match(value: str, result: tuple[int, int, str]) -> None:
-    assert find_replace_part(value, 0) == result
+def test_match_expr(value: str, exp_output: list[str | MatchExpression]) -> None:
+    assert find_replace_expr(value) == exp_output
+
+
+@pytest.mark.parametrize(
+    ("value", "exp_exception"),
+    [
+        ("py-{foo,bar}", None),
+        ("py37-{base,i18n},b", None),
+        ("py37-{i18n,base},b", None),
+        ("{toxinidir,}", None),
+        ("{env}", r"MatchError\('No variable name was supplied in {env} substitution'\)"),
+    ],
+)
+def test_dont_replace(replace_one: ReplaceOne, value: str, exp_exception: str | None) -> None:
+    """Test that invalid expressions are not replaced."""
+    if exp_exception:
+        with pytest.raises(HandledError, match=exp_exception):
+            replace_one(value)
+    else:
+        assert replace_one(value) == value
+
+
+@pytest.mark.parametrize(
+    ("match_expression", "exp_repr"),
+    [
+        (MatchExpression([["posargs"]]), "MatchExpression(expr=[['posargs']], term_pos=None)"),
+        (MatchExpression([["posargs"]], 1), "MatchExpression(expr=[['posargs']], term_pos=1)"),
+        (MatchExpression("foo", -42), "MatchExpression(expr='foo', term_pos=-42)"),
+    ],
+)
+def test_match_expression_repr(match_expression: MatchExpression, exp_repr: str) -> None:
+    print(match_expression)
+    assert repr(match_expression) == exp_repr

tests/config/loader/ini/replace/test_replace_env_var.py

@@ -1,5 +1,10 @@
 from __future__ import annotations
 
+import threading
+from typing import Generator
+
+import pytest
+
 from tests.config.loader.ini.replace.conftest import ReplaceOne
 from tox.pytest import MonkeyPatch
 
@@ -11,6 +16,43 @@ def test_replace_env_set(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> N
     assert result == "something good"
 
 
+def test_replace_env_set_double_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Double backslash should escape to single backslash and not affect surrounding replacements."""
+    monkeypatch.setenv("MAGIC", "something good")
+    result = replace_one(r"{env:MAGIC}\\{env:MAGIC}")
+    assert result == r"something good\something good"
+
+
+def test_replace_env_set_triple_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Triple backslash should escape to single backslash also escape subsequent replacement."""
+    monkeypatch.setenv("MAGIC", "something good")
+    result = replace_one(r"{env:MAGIC}\\\{env:MAGIC}")
+    assert result == r"something good\{env:MAGIC}"
+
+
+def test_replace_env_set_quad_bs(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Quad backslash should escape to two backslashes and not affect surrounding replacements."""
+    monkeypatch.setenv("MAGIC", "something good")
+    result = replace_one(r"\\{env:MAGIC}\\\\{env:MAGIC}\\")
+    assert result == r"\something good\\something good" + "\\"
+
+
+def test_replace_env_when_value_is_backslash(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """When the replacement value is backslash, it shouldn't affect the next replacement."""
+    monkeypatch.setenv("MAGIC", "tragic")
+    monkeypatch.setenv("BS", "\\")
+    result = replace_one(r"{env:BS}{env:MAGIC}")
+    assert result == r"\tragic"
+
+
+def test_replace_env_when_value_is_stuff_then_backslash(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """When the replacement value is a string containing backslash, it shouldn't affect the next replacement."""
+    monkeypatch.setenv("MAGIC", "tragic")
+    monkeypatch.setenv("BS", "stuff\\")
+    result = replace_one(r"{env:BS}{env:MAGIC}")
+    assert result == r"stuff\tragic"
+
+
 def test_replace_env_missing(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
     """If we have a factor that is not specified within the core env-list then that's also an environment"""
     monkeypatch.delenv("MAGIC", raising=False)
@@ -34,14 +76,60 @@ def test_replace_env_missing_default_from_env(replace_one: ReplaceOne, monkeypat
 
 
 def test_replace_env_var_circular(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
-    """If we have a factor that is not specified within the core env-list then that's also an environment"""
+    """Replacement values will not infinitely loop"""
     monkeypatch.setenv("MAGIC", "{env:MAGIC}")
     result = replace_one("{env:MAGIC}")
     assert result == "{env:MAGIC}"
 
 
+@pytest.fixture()
+def reset_env_var_after_delay(monkeypatch: MonkeyPatch) -> Generator[threading.Thread, None, None]:
+    timeout = 2
+
+    def avoid_infinite_loop() -> None:  # pragma: no cover
+        monkeypatch.setenv("TRAGIC", f"envvar forcibly reset after {timeout} sec")
+
+    timer = threading.Timer(2, avoid_infinite_loop)
+    timer.start()
+    yield timer
+    timer.cancel()
+    timer.join()
+
+
+@pytest.mark.usefixtures("reset_env_var_after_delay")
+def test_replace_env_var_circular_flip_flop(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Replacement values will not infinitely loop back and forth"""
+    monkeypatch.setenv("TRAGIC", "{env:MAGIC}")
+    monkeypatch.setenv("MAGIC", "{env:TRAGIC}")
+    result = replace_one("{env:MAGIC}")
+    assert result == "{env:TRAGIC}"
+
+
+@pytest.mark.parametrize("fallback", [True, False])
+def test_replace_env_var_chase(replace_one: ReplaceOne, monkeypatch: MonkeyPatch, fallback: bool) -> None:
+    """Resolve variable to be replaced and default value via indirection."""
+    monkeypatch.setenv("WALK", "THIS")
+    def_val = "or that one"
+    monkeypatch.setenv("DEF", def_val)
+    if fallback:
+        monkeypatch.delenv("THIS", raising=False)
+        exp_result = def_val
+    else:
+        this_val = "path"
+        monkeypatch.setenv("THIS", this_val)
+        exp_result = this_val
+    result = replace_one("{env:{env:WALK}:{env:DEF}}")
+    assert result == exp_result
+
+
 def test_replace_env_default_with_colon(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
     """If we have a factor that is not specified within the core env-list then that's also an environment"""
     monkeypatch.delenv("MAGIC", raising=False)
     result = replace_one("{env:MAGIC:https://some.url.org}")
     assert result == "https://some.url.org"
+
+
+def test_replace_env_default_deep(replace_one: ReplaceOne, monkeypatch: MonkeyPatch) -> None:
+    """Get the value through a long tree of nested defaults."""
+    monkeypatch.delenv("M", raising=False)
+    assert replace_one("{env:M:{env:M:{env:M:{env:M:{env:M:foo}}}}}") == "foo"

tests/config/loader/ini/replace/test_replace_os_sep.py

@@ -2,9 +2,29 @@
 
 import os
 
+import pytest
+
 from tests.config.loader.ini.replace.conftest import ReplaceOne
+from tox.pytest import MonkeyPatch
 
 
 def test_replace_os_sep(replace_one: ReplaceOne) -> None:
     result = replace_one("{/}")
     assert result == os.sep
+
+
+@pytest.mark.parametrize("sep", ["/", "\\"])
+def test_replace_os_sep_before_curly(monkeypatch: MonkeyPatch, replace_one: ReplaceOne, sep: str) -> None:
+    """Explicit test case for issue #2732 (windows only)."""
+    monkeypatch.setattr(os, "sep", sep)
+    monkeypatch.delenv("_", raising=False)
+    result = replace_one("{/}{env:_:foo}")
+    assert result == os.sep + "foo"
+
+
+@pytest.mark.parametrize("sep", ["/", "\\"])
+def test_replace_os_sep_sub_exp_regression(monkeypatch: MonkeyPatch, replace_one: ReplaceOne, sep: str) -> None:
+    monkeypatch.setattr(os, "sep", sep)
+    monkeypatch.delenv("_", raising=False)
+    result = replace_one("{env:_:{posargs}{/}.{posargs}}", ["foo"])
+    assert result == f"foo{os.sep}.foo"