Fix crash if dependencies have prerelease requires-python versions (#…

…1112)

* Fix crash if dependencies have prerelease requires-python versions

Recently, coverage 6.4.1 listed its requirements with an alpha version of
python and this broke PDM's version parsing:

    extras_require={
        'toml': ['tomli; python_full_version<="3.11.0a6"'],
    },

Prerelease `requires-python` versions *are* valid, per these specifications:

* https://peps.python.org/pep-0440/
* https://peps.python.org/pep-0621/#requires-python
* https://packaging.python.org/en/latest/specifications/core-metadata/#requires-python

Therefore this commit adds missing parsing support for `{a|b|rc}[N]`
pre-release specifiers which are used by python language releases:

* https://docs.python.org/3/faq/general.html#how-does-the-python-version-numbering-scheme-work

This bug meant that projects that directly or indirectly depended on coverage
were unable to update pdm.lock using commands like `update` `add` `lock` and
`install` because `pdm.models.versions.Version` would raise:

    pdm.exceptions.InvalidPyVersion: 3.11.0a6: Prereleases or postreleases are not supported for python version specifers.

Until this is fixed, projects can workaround this by depending on:

    "coverage<6.4",
    "coverage[toml]<6.4",

Fixes pdm-project/pdm#1111

* Remove workaround for pre-release python in global environment

Since prereleases are now supported in python specifiers (in the
previous commit), we can remove the workaround (from commit 8e7260968ac
for #932) that converted a python prerelease version to a specifier
range.

Co-authored-by: Hashem Nasarat <hnasarat@beta.team>

news/1111.bugfix.md

@@ -0,0 +1 @@
+Fix a bug where dependencies with `requires-python` pre-release versions caused `pdm update` to fail with `InvalidPyVersion`.

pdm/models/specifiers.py

@@ -6,7 +6,6 @@
 from pathlib import Path
 from typing import Any, Iterable, List, Set, Tuple, Union, cast
 
-from packaging.version import Version as PackageVersion
 from pip._vendor.packaging.specifiers import SpecifierSet
 
 from pdm.exceptions import InvalidPyVersion
@@ -69,8 +68,6 @@ def __init__(self, version_str: str = "", analyze: bool = True) -> None:
             self._analyze_specifiers()
 
     def _analyze_specifiers(self) -> None:
-        # XXX: Prerelease or postrelease specifiers will fail here, but I guess we can
-        # just ignore them for now.
         lower_bound, upper_bound = Version.MIN, Version.MAX
         excludes: Set[Version] = set()
         for spec in self:
@@ -101,19 +98,6 @@ def _analyze_specifiers(self) -> None:
                 raise InvalidPyVersion(f"Unsupported version specifier: {op}{version}")
         self._rearrange(lower_bound, upper_bound, excludes)
 
-    @classmethod
-    def equal_to(cls, version: PackageVersion) -> "PySpecSet":
-        """Create a specifierset that is equal to the given version."""
-        if not version.is_prerelease:
-            return cls(f"=={version}")
-        spec = cls(f"=={version}", analyze=False)
-        spec._upper_bound = Version((version.major, version.minor, 0))
-        lower_bound = Version((version.major, version.minor - 1))
-        spec._lower_bound = lower_bound.complete(
-            cls.PY_MAX_MINOR_VERSION[lower_bound] + 1
-        )
-        return spec
-
     @classmethod
     def _merge_bounds_and_excludes(
         cls,
@@ -238,9 +222,9 @@ def __str__(self) -> str:
             return ""
         lower = self._lower_bound
         upper = self._upper_bound
-        if lower[-1] == 0:
+        if lower[-1] == 0 and not lower.is_prerelease:
             lower = lower[:-1]
-        if upper[-1] == 0:
+        if upper[-1] == 0 and not upper.is_prerelease:
             upper = upper[:-1]
         lower_str = "" if lower == Version.MIN else f">={lower}"
         upper_str = "" if upper == Version.MAX else f"<{upper}"

pdm/models/versions.py

@@ -1,37 +1,56 @@
 import re
-from typing import Any, Tuple, Union, cast, overload
+from typing import Any, List, Optional, Tuple, Union, overload
 
 from pdm._types import Literal
 from pdm.exceptions import InvalidPyVersion
 
 VersionBit = Union[int, Literal["*"]]
+PRE_RELEASE_SEGMENT_RE = re.compile(
+    r"(?P<digit>\d+)(?P<type>a|b|rc)(?P<n>\d*)",
+    flags=re.IGNORECASE,
+)
 
 
 class Version:
     """A loosely semantic version implementation that allows '*' in version part.
 
     This class is designed for Python specifier set merging only, hence up to 3 version
-    parts are kept, plus prereleases or postreleases are not supported.
+    parts are kept, plus optional prerelease suffix.
+
+    This is a slightly different purpose than packaging.version.Version which is
+    focused on supporting PEP 440 version identifiers, not specifiers.
     """
 
     MIN: "Version"
     MAX: "Version"
+    # Pre-release may follow version with {a|b|rc}N
+    # https://docs.python.org/3/faq/general.html#how-does-the-python-version-numbering-scheme-work
+    pre: Optional[Tuple[str, int]] = None
 
     def __init__(self, version: Union[Tuple[VersionBit, ...], str]) -> None:
         if isinstance(version, str):
             version_str = re.sub(r"(?<!\.)\*", ".*", version)
-            try:
-                version = cast(
-                    Tuple[VersionBit, ...],
-                    tuple(int(v) if v != "*" else v for v in version_str.split("."))[
-                        :3
-                    ],
-                )
-            except ValueError:
-                raise InvalidPyVersion(
-                    f"{version_str}: Prereleases or postreleases are not supported "
-                    "for python version specifers."
-                )
+            bits: List[VersionBit] = []
+            for v in version_str.split(".")[:3]:
+                try:
+                    bits.append(int(v))
+                except ValueError:
+                    pre_m = PRE_RELEASE_SEGMENT_RE.match(v)
+                    if v == "*":
+                        bits.append("*")
+                        break  # .* is only allowed at the end, per PEP 440
+                    elif pre_m:
+                        bits.append(int(pre_m.group("digit")))
+                        pre_type = pre_m.group("type").lower()
+                        pre_n = int(pre_m.group("n") or "0")
+                        self.pre = (pre_type, pre_n)
+                        break  # pre release version is only at the end
+                    else:
+                        raise InvalidPyVersion(
+                            f"{version_str}: postreleases are not supported "
+                            "for python version specifiers."
+                        )
+            version = tuple(bits)
         self._version: Tuple[VersionBit, ...] = version
 
     def complete(self, complete_with: VersionBit = 0, max_bits: int = 3) -> "Version":
@@ -40,15 +59,24 @@ def complete(self, complete_with: VersionBit = 0, max_bits: int = 3) -> "Version
         """
         assert len(self._version) <= max_bits, self
         new_tuple = self._version + (max_bits - len(self._version)) * (complete_with,)
-        return type(self)(new_tuple)
+        ret = type(self)(new_tuple)
+        ret.pre = self.pre
+        return ret
 
     def bump(self, idx: int = -1) -> "Version":
         """Bump version by incrementing 1 on the given index of version part.
-        Increment the last version bit by default.
+        If index is not provided: increment the last version bit unless version
+        is a pre-release, in which case, increment the pre-release number.
         """
         version = self._version
-        head, value = version[:idx], int(version[idx])
-        return type(self)((*head, value + 1)).complete()
+        if idx == -1 and self.pre:
+            ret = type(self)(version).complete()
+            ret.pre = (self.pre[0], self.pre[1] + 1)
+        else:
+            head, value = version[:idx], int(version[idx])
+            ret = type(self)((*head, value + 1)).complete()
+            ret.pre = None
+        return ret
 
     def startswith(self, other: "Version") -> bool:
         """Check if the version begins with another version."""
@@ -59,23 +87,41 @@ def is_wildcard(self) -> bool:
         """Check if the version ends with a '*'"""
         return self._version[-1] == "*"
 
+    @property
+    def is_prerelease(self) -> bool:
+        """Check if the version is a prerelease."""
+        return self.pre is not None
+
     def __str__(self) -> str:
-        return ".".join(map(str, self._version))
+        parts = []
+        parts.append(".".join(map(str, self._version)))
+
+        if self.pre:
+            parts.append("".join(str(x) for x in self.pre))
+
+        return "".join(parts)
 
     def __repr__(self) -> str:
         return f"<Version({self})>"
 
     def __eq__(self, other: Any) -> bool:
         if not isinstance(other, Version):
             return NotImplemented
-        return self._version == other._version
+        return self._version == other._version and self.pre == other.pre
 
     def __lt__(self, other: Any) -> bool:
         if not isinstance(other, Version):
             return NotImplemented
 
-        def comp_key(version: Version) -> Tuple[int, ...]:
-            return tuple(-1 if v == "*" else v for v in version._version)
+        def comp_key(version: Version) -> List[float]:
+            ret: List[float] = [-1 if v == "*" else v for v in version._version]
+            if version.pre:
+                # Get the ascii value of first character, a < b < r[c]
+                ret += [ord(version.pre[0][0]), version.pre[1]]
+            else:
+                ret += [float("inf")]
+
+            return ret
 
         return comp_key(self) < comp_key(other)
 
@@ -110,7 +156,7 @@ def __setitem__(self, idx: int, value: VersionBit) -> None:
         self._version = tuple(version)
 
     def __hash__(self) -> int:
-        return hash(self._version)
+        return hash((self._version, self.pre))
 
     @property
     def is_py2(self) -> bool:

pdm/project/core.py

@@ -241,7 +241,7 @@ def get_environment(self) -> Environment:
             env = GlobalEnvironment(self)
             # Rewrite global project's python requires to be
             # compatible with the exact version
-            env.python_requires = PySpecSet.equal_to(self.python.version)
+            env.python_requires = PySpecSet(f"=={self.python.version}")
             return env
         if self.config["python.use_venv"] and get_venv_like_prefix(
             self.python.executable

tests/models/test_specifiers.py

@@ -23,6 +23,8 @@
         (">3.4.*", ">=3.5"),
         ("<=3.4.*", "<3.4"),
         ("<3.4.*", "<3.4"),
+        ("<3.10.0a6", "<3.10.0a6"),
+        ("<3.10.2a3", "<3.10.2a3"),
     ],
 )
 def test_normalize_pyspec(original, normalized):
@@ -38,6 +40,8 @@ def test_normalize_pyspec(original, normalized):
         ("", ">=3.6", ">=3.6"),
         (">=3.6", "<3.2", "impossible"),
         (">=2.7,!=3.0.*", "!=3.1.*", ">=2.7,!=3.0.*,!=3.1.*"),
+        (">=3.11.0a2", "<3.11.0b", ">=3.11.0a2,<3.11.0b0"),
+        ("<3.11.0a2", ">3.11.0b", "impossible"),
     ],
 )
 def test_pyspec_and_op(left, right, result):
@@ -55,6 +59,7 @@ def test_pyspec_and_op(left, right, result):
         (">=3.6,<3.8", ">=3.4,<3.7", ">=3.4,<3.8"),
         ("~=2.7", ">=3.6", ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*"),
         ("<2.7.15", ">=3.0", "!=2.7.15,!=2.7.16,!=2.7.17,!=2.7.18"),
+        (">3.11.0a2", ">3.11.0b", ">=3.11.0a3"),
     ],
 )
 def test_pyspec_or_op(left, right, result):
@@ -86,6 +91,7 @@ def test_impossible_pyspec():
             ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*",
             ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*",
         ),
+        (">=3.11*", ">=3.11.0rc"),  # 11* normalizes to 11.0
     ],
 )
 def test_pyspec_is_subset_superset(left, right):
@@ -102,6 +108,7 @@ def test_pyspec_is_subset_superset(left, right):
         (">=3.7", ">=3.6,<3.9"),
         (">=3.7,<3.6", "==2.7"),
         (">=3.0,!=3.4.*", ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*"),
+        (">=3.11.0", "<3.11.0a"),
     ],
 )
 def test_pyspec_isnot_subset_superset(left, right):

tests/models/test_versions.py

@@ -3,9 +3,19 @@
 from pdm.models.versions import InvalidPyVersion, Version
 
 
-def test_unsupported_prerelease_version():
+def test_unsupported_post_version() -> None:
     with pytest.raises(InvalidPyVersion):
-        Version("3.9.0a4")
+        Version("3.10.0post1")
+
+
+def test_support_prerelease_version() -> None:
+    assert not Version("3.9.0").is_prerelease
+    v = Version("3.9.0a4")
+    assert v.is_prerelease
+    assert str(v) == "3.9.0a4"
+    assert v.complete() == v
+    assert v.bump() == Version("3.9.0a5")
+    assert v.bump(2) == Version("3.9.1")
 
 
 def test_normalize_non_standard_version():
@@ -19,6 +29,12 @@ def test_version_comparison():
     assert Version("3.7.*") < Version("3.7.5")
     assert Version("3.7") == Version((3, 7))
 
+    assert Version("3.9.0a") != Version("3.9.0")
+    assert Version("3.9.0a") == Version("3.9.0a0")
+    assert Version("3.10.0a9") < Version("3.10.0a12")
+    assert Version("3.10.0a12") < Version("3.10.0b1")
+    assert Version("3.7.*") < Version("3.7.1b")
+
 
 def test_version_is_wildcard():
     assert not Version("3").is_wildcard