Environment markers with different sources for a dependency #8303

sassanh · 2023-08-12T21:47:11Z

Poetry version: 1.5.1
Python version: 3.11.4
OS version and name: macOS Sonoma 14.0
pyproject.toml: -

I am on the latest stable Poetry version, installed using a recommended method.
I have searched the issues of this repo and believe that this is not a duplicate.
I have consulted the FAQ and blog for any relevant entries or release notes.
If an exception occurs when executing a command, I executed it again in debug mode (-vvv option) and have included the output below.

Issue

I have read #7300, #5205 and #6710. I discussed it in Discord but didn't get any answer. The issue is if I need to install a package from different sources based on the platform, poetry doesn't behave as expected.
So imagine I have this dependency:

[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"

[tool.poetry.dependencies]
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = "piwheels" },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = "PyPI" }
]

I expect it to install from piwheels on RPi and install from PyPI on my macOS, but it doesn't. It creates this in lock file:

[[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5aa8a0ca0708dd6edb0127c91712db7b8aad3aea3a79d498df145b8a7ee0aee6"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af45f433208c16a4d0e755fafac5f2238560880a475a8a041ca9d99695a6b166"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:564705e8fbd788dce22dae8dfe0488be705e14381fa59d6bdfa3df5255446429"},
    {file = "Kivy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:19cd0b0b40046d6233f0a30c8003e1ffe5a135b96ae50a5442bd62ef9d9c6744"},
    {file = "Kivy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:4915e931cb26efa016956b16a0d04bcc8eb89abcb2734ed718786c30241795bb"},
    {file = "Kivy-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e3df4f6230f4b125c78b1dd923f714e93341afda5b55ae9795df2d6b2c5dbc4d"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eb5de5ffea53a341790ff7ba4d3cea60d37523f0766a4783692e2aa1d50670e"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a5e292f232a67bc67dfa91a205aed472dc4df7329ce059c74a88031c652efa"},
    {file = "Kivy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:d78a8d92565c6309bb1edc9a0c99883630adc0503dba25f3eb8b667ceb736395"},
    {file = "Kivy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:e317a710e0be3083901d29457fa8a203c14a71b7cef8008e3cdc7d1eb8e4d7c5"},
    {file = "Kivy-2.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7b8c8725cd1b3575b95f8f8b5d6f804e096af8958156450bb876821fbb2ce37"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b89bc5add63bf83fd1b026ec4d492ea9c90067b30652c14afe97cd4c30a2b5a"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f19ed8efaea7c880ab6342269c41ac47989812cbd036a524abc49ba9923487f"},
    {file = "Kivy-2.2.1-cp37-cp37m-win32.whl", hash = "sha256:c932fa9a8ade002ec4468c416bedda4188cda53721be84e4086b278c3b00c654"},
    {file = "Kivy-2.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7eeda699fe764c5efa2cc43e0ef28275be2a56ccd8af17933b6e2457d66d5b0d"},
    {file = "Kivy-2.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0456359afe01a94d39bb29ce8a8dd9d9dcd86cec6532f17bffb0725647a8d63a"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37874a7a2a9983ae494910578e2e6a4aba1568b35f915b014d85f779b5add8b9"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7bfbc97f66d50d71bcbc5c254e74386d94d7a1fb051448fbc1ba6742c1d1d21"},
    {file = "Kivy-2.2.1-cp38-cp38-win32.whl", hash = "sha256:0454ad509b83f5ee04dd7bb46682f077adcde99d86c206127f2dd57aedd28f5b"},
    {file = "Kivy-2.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:a6b47a8b5b47d091ac4604635edef7522ddfb9cf47b3f258cccd8ffbf094b429"},
    {file = "Kivy-2.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0e8370d529548d360803424f803231ed8ad19c7f4c351aa893fe9d39bcc940fc"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1e9cf59dc3875c943c89d3668a73235c96f4c03680c4691416ccc5d1c7f50f0"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97442522fb58ebf4038f3c57a795046e00a23f7c4dc0775b282af9aff845732a"},
    {file = "Kivy-2.2.1-cp39-cp39-win32.whl", hash = "sha256:f3dcfdce8403dc3df7a8b1a7d424c0939d41bc48a0b627403b685931a609a7ec"},
    {file = "Kivy-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:35eb3025a77894380196d5d64852da6c539e72aa2ff4970011385eb7a70c46e0"},
    {file = "Kivy-2.2.1.tar.gz", hash = "sha256:047a434e8efd32d425321e065fcccf725f165adb0c81ed69dc33c360d4796f7e"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

Note that markers are completely ignored and dropped in the lock file and the package is mentioned two times.

Now this lock file works on RPi but doesn't work on macOS, if I change the order of sources in my pyproject.toml it will affect the order of packages in the lock file as well and then it will work on macOS but not in RPi.

The text was updated successfully, but these errors were encountered:

sassanh · 2023-08-12T21:53:33Z

Another example, if I have this dependency:

[tool.poetry.dependencies]
rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

to install this package only in RPi and not in other platforms, I see this in the lock file:

[[package]]
name = "rpi-gpio"
version = "0.7.1"
description = "A module to control Raspberry Pi GPIO channels"
optional = false
python-versions = "*"
files = [
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv6l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv7l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

Ignoring the markers, so basically this:

[tool.poetry.dependencies]
rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

behaves the same as

[tool.poetry.dependencies]
rpi-gpio = "^0.7.1"

dimbleby · 2023-08-13T01:26:35Z

looks as though poetry forgets about what repository a package comes from when it re-solves from its lockfile during installation.

A patch something like this should do it:

--- a/src/poetry/installation/installer.py
+++ b/src/poetry/installation/installer.py
@@ -286,16 +286,20 @@ class Installer:
             )

         # We resolve again by only using the lock file
-        pool = RepositoryPool(ignore_repository_names=True, config=self._config)
+        pool = RepositoryPool(config=self._config)

-        # Making a new repo containing the packages
-        # newly resolved and the ones from the current lock file
-        repo = Repository("poetry-repo")
+        # Make a new collection of repositories containing the packages newly resolved
+        # and the ones from the current lock file.
         for package in lockfile_repo.packages + locked_repository.packages:
-            if not package.is_direct_origin() and not repo.has_package(package):
-                repo.add_package(package)
+            if not package.is_direct_origin():
+                repo_name = package.source_reference or "PyPI"
+                try:
+                    repo = pool.repository(repo_name)
+                except IndexError:
+                    repo = Repository(repo_name)
+                    pool.add_repository(repo)

-        pool.add_repository(repo)
+                repo.add_package(package)

The dull bit will be figuring out a unit test which I've no intention of doing! If this is important to you - or anyone else - then contribution surely welcome.

sassanh · 2023-08-13T10:24:54Z

Thanks! This fixes dropping sources in the lock file, but it still ignores markers in the lock file, so this is the relevant part of the generated lock file:

[[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:5aa8a0ca0708dd6edb0127c91712db7b8aad3aea3a79d498df145b8a7ee0aee6"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:af45f433208c16a4d0e755fafac5f2238560880a475a8a041ca9d99695a6b166"},
    {file = "Kivy-2.2.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:564705e8fbd788dce22dae8dfe0488be705e14381fa59d6bdfa3df5255446429"},
    {file = "Kivy-2.2.1-cp310-cp310-win32.whl", hash = "sha256:19cd0b0b40046d6233f0a30c8003e1ffe5a135b96ae50a5442bd62ef9d9c6744"},
    {file = "Kivy-2.2.1-cp310-cp310-win_amd64.whl", hash = "sha256:4915e931cb26efa016956b16a0d04bcc8eb89abcb2734ed718786c30241795bb"},
    {file = "Kivy-2.2.1-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:e3df4f6230f4b125c78b1dd923f714e93341afda5b55ae9795df2d6b2c5dbc4d"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0eb5de5ffea53a341790ff7ba4d3cea60d37523f0766a4783692e2aa1d50670e"},
    {file = "Kivy-2.2.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42a5e292f232a67bc67dfa91a205aed472dc4df7329ce059c74a88031c652efa"},
    {file = "Kivy-2.2.1-cp311-cp311-win32.whl", hash = "sha256:d78a8d92565c6309bb1edc9a0c99883630adc0503dba25f3eb8b667ceb736395"},
    {file = "Kivy-2.2.1-cp311-cp311-win_amd64.whl", hash = "sha256:e317a710e0be3083901d29457fa8a203c14a71b7cef8008e3cdc7d1eb8e4d7c5"},
    {file = "Kivy-2.2.1-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a7b8c8725cd1b3575b95f8f8b5d6f804e096af8958156450bb876821fbb2ce37"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b89bc5add63bf83fd1b026ec4d492ea9c90067b30652c14afe97cd4c30a2b5a"},
    {file = "Kivy-2.2.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f19ed8efaea7c880ab6342269c41ac47989812cbd036a524abc49ba9923487f"},
    {file = "Kivy-2.2.1-cp37-cp37m-win32.whl", hash = "sha256:c932fa9a8ade002ec4468c416bedda4188cda53721be84e4086b278c3b00c654"},
    {file = "Kivy-2.2.1-cp37-cp37m-win_amd64.whl", hash = "sha256:7eeda699fe764c5efa2cc43e0ef28275be2a56ccd8af17933b6e2457d66d5b0d"},
    {file = "Kivy-2.2.1-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0456359afe01a94d39bb29ce8a8dd9d9dcd86cec6532f17bffb0725647a8d63a"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37874a7a2a9983ae494910578e2e6a4aba1568b35f915b014d85f779b5add8b9"},
    {file = "Kivy-2.2.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b7bfbc97f66d50d71bcbc5c254e74386d94d7a1fb051448fbc1ba6742c1d1d21"},
    {file = "Kivy-2.2.1-cp38-cp38-win32.whl", hash = "sha256:0454ad509b83f5ee04dd7bb46682f077adcde99d86c206127f2dd57aedd28f5b"},
    {file = "Kivy-2.2.1-cp38-cp38-win_amd64.whl", hash = "sha256:a6b47a8b5b47d091ac4604635edef7522ddfb9cf47b3f258cccd8ffbf094b429"},
    {file = "Kivy-2.2.1-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0e8370d529548d360803424f803231ed8ad19c7f4c351aa893fe9d39bcc940fc"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d1e9cf59dc3875c943c89d3668a73235c96f4c03680c4691416ccc5d1c7f50f0"},
    {file = "Kivy-2.2.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:97442522fb58ebf4038f3c57a795046e00a23f7c4dc0775b282af9aff845732a"},
    {file = "Kivy-2.2.1-cp39-cp39-win32.whl", hash = "sha256:f3dcfdce8403dc3df7a8b1a7d424c0939d41bc48a0b627403b685931a609a7ec"},
    {file = "Kivy-2.2.1-cp39-cp39-win_amd64.whl", hash = "sha256:35eb3025a77894380196d5d64852da6c539e72aa2ff4970011385eb7a70c46e0"},
    {file = "Kivy-2.2.1.tar.gz", hash = "sha256:047a434e8efd32d425321e065fcccf725f165adb0c81ed69dc33c360d4796f7e"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[[package]]
name = "kivy"
version = "2.2.1"
description = "A software library for rapid development of hardware-accelerated multitouch applications."
optional = false
python-versions = ">=3.7"
files = [
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:f0312dc2b154f2730acaaac1fb9b60081dc55026b2c861fd1d9e34834b5d71ff"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
    {file = "Kivy-2.2.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:61e5dd7ea9d03a6ed8dcd735328049ca1ccb5d7ed5db8db6215b8106115e832a"},
]

[package.dependencies]
docutils = "*"
"kivy-deps.angle" = {version = ">=0.3.3,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.glew" = {version = ">=0.3.1,<0.4.0", markers = "sys_platform == \"win32\""}
"kivy-deps.sdl2" = {version = ">=0.6.0,<0.7.0", markers = "sys_platform == \"win32\""}
Kivy-Garden = ">=0.1.4"
pygments = "*"
pypiwin32 = {version = "*", markers = "sys_platform == \"win32\""}

[package.extras]
angle = ["kivy-deps.angle (>=0.3.3,<0.4.0)"]
base = ["docutils", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32", "requests"]
dev = ["flake8", "funcparserlib (==1.0.0a0)", "kivy-deps.glew-dev (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer-dev (>=0.3.3,<0.4.0)", "kivy-deps.sdl2-dev (>=0.6.0,<0.7.0)", "pre-commit", "pyinstaller", "pytest (>=3.6)", "pytest-asyncio (!=0.11.0)", "pytest-benchmark", "pytest-cov", "pytest-timeout", "responses", "sphinx (<=6.2.1)", "sphinxcontrib-actdiag", "sphinxcontrib-blockdiag", "sphinxcontrib-jquery", "sphinxcontrib-nwdiag", "sphinxcontrib-seqdiag"]
full = ["docutils", "ffpyplayer", "kivy-deps.angle (>=0.3.3,<0.4.0)", "kivy-deps.glew (>=0.3.1,<0.4.0)", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)", "kivy-deps.sdl2 (>=0.6.0,<0.7.0)", "pillow", "pygments", "pypiwin32"]
glew = ["kivy-deps.glew (>=0.3.1,<0.4.0)"]
gstreamer = ["kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
media = ["ffpyplayer", "kivy-deps.gstreamer (>=0.3.3,<0.4.0)"]
sdl2 = ["kivy-deps.sdl2 (>=0.6.0,<0.7.0)"]
tuio = ["oscpy"]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

Since markers are dropped, it tries to install from wrong source (PyPI) on RPi.

And for rpi-gpio with this dependency entry:

rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

this is the generated lock file:

[[package]]
name = "rpi-gpio"
version = "0.7.1"
description = "A module to control Raspberry Pi GPIO channels"
optional = false
python-versions = "*"
files = [
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv6l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv7l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

Again since markers are dropped it tries to install this package on macOS.

dimbleby · 2023-08-13T10:47:30Z

the lockfile is fine, it doesn't need additional markers. I don't have a mac but I confirm that on my non-aarch64 linux VM the above requirement does not cause rpi-gpio to be installed

sassanh · 2023-08-14T12:11:49Z

For me it clearly tries to install rpi-gpio on macOS and shows me this error:

Installing dependencies from lock file

Package operations: 3 installs, 0 updates, 0 removals

  • Installing rpi-gpio (0.7.1): Failed

  RuntimeError

  Unable to find installation candidates for rpi-gpio (0.7.1)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:76 in choose_for
       72│ 
       73│             links.append(link)
       74│ 
       75│         if not links:
    →  76│             raise RuntimeError(f"Unable to find installation candidates for {package}")
       77│ 
       78│         # Get the best link
       79│         chosen = max(links, key=lambda link: self._sort_key(package, link))
       80│ 

  • Installing rpi-ws281x (5.0.0): Failed

  RuntimeError

  Unable to find installation candidates for rpi-ws281x (5.0.0)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:76 in choose_for
       72│ 
       73│             links.append(link)
       74│ 
       75│         if not links:
    →  76│             raise RuntimeError(f"Unable to find installation candidates for {package}")
       77│ 
       78│         # Get the best link
       79│         chosen = max(links, key=lambda link: self._sort_key(package, link))
       80│ 

  • Installing sysv-ipc (1.1.0): Failed

  RuntimeError

  Unable to find installation candidates for sysv-ipc (1.1.0)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:76 in choose_for
       72│ 
       73│             links.append(link)
       74│ 
       75│         if not links:
    →  76│             raise RuntimeError(f"Unable to find installation candidates for {package}")
       77│ 
       78│         # Get the best link
       79│         chosen = max(links, key=lambda link: self._sort_key(package, link))
       80│

I even tried the 3 commits in the linked pull request.

Other than that I wonder how you consider this lockfile fine when it doesn't have the markers? How does the installer know it shouldn't install rpi-gpio without the markers?

sassanh · 2023-08-14T12:31:56Z

I noticed when I put sources in this order:

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"

[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

it generates a lock file which doesn't try to install rpi-gpio on macOS but if I change the order it generates a lock file which tries to install rpi-gpio and fails. What is confusing for me is that the relevant part for rpi-gpio is exactly the same in both lock files which is:

[[package]]
name = "rpi-gpio"
version = "0.7.1"
description = "A module to control Raspberry Pi GPIO channels"
optional = false
python-versions = "*"
files = [
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv6l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp35-cp35m-linux_armv7l.whl", hash = "sha256:5073d1972727cfad38a1a42f02da91dc41f137679290aa804ab6c410168347ac"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv6l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp37-cp37m-linux_armv7l.whl", hash = "sha256:e5dfec2c4ccb7dbe9eef3dfc76a3b04121fc62d77fd8fbe1a8e841468a1b0c4c"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv6l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
    {file = "RPi.GPIO-0.7.1-cp39-cp39-linux_armv7l.whl", hash = "sha256:231f6142c25975a7e39b96faf4905f05f97dba6809e8c5f0d58f284b35b4b821"},
]

[package.source]
type = "legacy"
url = "https://www.piwheels.org/simple"
reference = "piwheels"

dimbleby · 2023-08-14T12:49:22Z

your mental model is wrong, the lockfile should be thought of more like a repository of available packages than a list of instructions. At install time poetry re-solves from that repository.

top-level requirements - including their markers - are listed in pyproject.toml

this certainly works just fine - poetry itself uses such a construction -

poetry/pyproject.toml

Line 61 in 2b16b2f

xattr = { version = "^0.10.0", markers = "sys_platform == 'darwin'" }

so I suppose you are either doing something more complicated than you have so far told us about; or the markers you have added just don't exclude the platform you are installing on.

sassanh · 2023-08-14T20:12:55Z

Thanks for the quick responses!

This is my complete pyproject.toml file:

[tool.poetry]
name = "headless-kivy-pi"
version = "0.2.0"
description = ""
authors = ["Sassan Haradji <sassanh@gmail.com>"]
license = "MIT"
readme = "README.md"
packages = [{ include = "headless_kivy_pi" }]


[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"


[tool.poetry.dependencies]
python = ">=3.9,<3.12"
numpy = "^1.24.2"
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]
typing-extensions = "^4.7.1"
adafruit-circuitpython-rgb-display = "^3.11.0"

rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }
adafruit-circuitpython-aw9523 = { version = "^1.1.7", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

fake-rpi = { version = "^0.7.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
watchdog = { version = "^3.0.0", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
screeninfo = { version = "^0.8.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }


[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

If I lock this file using poetry lock and then run poetry install it tries to install 3 packages meant to be installed for RPi including rpi-gpio and will fail with the above mentioned error.
If I change the order of sources like this:

[tool.poetry]
name = "headless-kivy-pi"
version = "0.2.0"
description = ""
authors = ["Sassan Haradji <sassanh@gmail.com>"]
license = "MIT"
readme = "README.md"
packages = [{ include = "headless_kivy_pi" }]


[[tool.poetry.source]]
name = "PyPI"
priority = "primary"

[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"


[tool.poetry.dependencies]
python = ">=3.9,<3.12"
numpy = "^1.24.2"
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]
typing-extensions = "^4.7.1"
adafruit-circuitpython-rgb-display = "^3.11.0"

rpi-gpio = { version = "^0.7.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' }
adafruit-circuitpython-aw9523 = { version = "^1.1.7", markers = "platform_machine == 'aarch64'", source = 'piwheels' }

fake-rpi = { version = "^0.7.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
watchdog = { version = "^3.0.0", markers = "platform_machine != 'aarch64'", source = 'PyPI' }
screeninfo = { version = "^0.8.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' }


[build-system]
requires = ["poetry-core"]
build-backend = "poetry.core.masonry.api"

And then lock it with poetry lock and run poetry install, it finishes its job successfully.

dimbleby · 2023-08-14T20:35:21Z

I expect rpi-gpio is an indirect dependency of one of your other requirements, but on all platforms.

Then, absent instructions to the contrary, poetry is happy to satisfy that from the piwheels repository .

sassanh · 2023-08-14T22:21:29Z

Yes, that was it, seems like adafruit-circuitpython-rgb-display has rpi-gpio as its dependency.

I really appreciate your help, so it should be safe to close this issue once the PR is merged.

sassanh · 2023-08-15T14:43:09Z

OK, now my package locks and installs the dependencies without any problems, you can see it here: https://github.com/sassanh/headless-kivy-pi
But when I try to install it by { path = "../headless-kivy-pi", develop = true } or { path = "../headless-kivy-pi/dist/....whl" } or { git = "https://github.com/sassanh/headless-kivy-pi.git" } it fails to install after locking when I have sources in this order

[[tool.poetry.source]]
name = "piwheels"
url = "https://www.piwheels.org/simple/"
priority = "primary"

[[tool.poetry.source]]
name = "PyPI"
priority = "primary"

and it installs successfully if I change the order of sources. It is on the macOS, things are opposite on the RPi (successful installation when phwheels is on top and failed installation when pypi is on top).

If I install the git repository via pip without using poetry at all, it just works without any errors, so I guess poetry deals dependencies with pyproject.toml differently and is not just pip-installing them.

The error message is something like this:

  • Installing numpy (1.25.2): Failed

  RuntimeError

  Unable to find installation candidates for numpy (1.25.2)

  at ~/Library/Application Support/pypoetry/venv/lib/python3.11/site-packages/poetry/installation/chooser.py:79 in choose_for
       75│ 
       76│             links.append(link)
       77│ 
       78│         if not links:
    →  79│             raise RuntimeError(
       80│                 f"Unable to find installation candidates for {package}")
       81│ 
       82│         # Get the best link
       83│         chosen = max(links, key=lambda link: self._sort_key(package, link))

when I print links when piwheels is on top I get this:

[<Link https://www.piwheels.org/simple/numpy/numpy-1.25.2-cp39-cp39-linux_armv7l.whl#sha256=4b0bf79e4c9ea416d3e7ddfd463263b7489c80e8fac9c94f3c9f11
ee443210b7 (requires-python:>=3.9)>, <Link https://www.piwheels.org/simple/numpy/numpy-1.25.2-cp39-cp39-linux_armv6l.whl#sha256=4b0bf79e4c9ea416d3
e7ddfd463263b7489c80e8fac9c94f3c9f11ee443210b7 (requires-python:>=3.9)>]```

so while the original issue is fixed for the dependencies, it is still an issue for the dependencies of the dependencies.

dimbleby · 2023-08-15T21:45:35Z

sources are a poetry-special, there is no way to pass them through standard metadata, it's expected that project can know nothing about sources defined in dependency.

if you search you will find other issues in this repository discussing this

sassanh · 2023-08-15T23:28:58Z

there is no way to pass them through standard metadata

It's good that sources of project A being a dependency of project B, doesn't have any effect on project B, what is happening here is that the order of sources of the project B, affects installation of project A (the reverse direction) if I can install A via pip successfully, I should be able to install it via poetry as well, and changing the order of sources in B shouldn't cause A to fail to install. Am I missing something here?

sassanh · 2023-08-15T23:32:59Z

So to be clear what is happening here is that I have project A, I can run poetry install in it and it runs successfully, I can build a wheel out of it and install the wheel using pip install and it installs successfully. But installing it via poetry sometimes fails and it depends on the order of sources in the pyproject.toml (not the one inside A).

To me it looks like a leak of information (and in this case a destructive leak of information) from the current project to its dependencies.

radoering · 2023-08-16T16:27:39Z

That's how it is. If project B depends on project A, project B will not know about any poetry-specific information in project A's pyproject.toml, e.g. explicit sources. You basically have to repeat explicit source constraints for dependencies where it's relevant in project B's pyproject.toml. I'd propose to create an additional (optional) dependency group for transitive dependencies in such cases. For example:

project A:

[tool.poetry.dependencies]
kivy = [
  { version = "^2.2.1", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "^2.2.1", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]

project B:

[tool.poetry.dependencies]
project-a = ...

[tool.poetry.group.transitive]
optional = true

[tool.poetry.group.transitive.dependencies]
kivy = [
  { version = "*", markers = "platform_machine == 'aarch64'", source = 'piwheels' },
  { version = "*", markers = "platform_machine != 'aarch64'", source = 'PyPI' },
]

I am aware that this may not be a satisfactory solution in this case but that's what Poetry's design allows.

github-actions · 2024-02-29T08:06:02Z

This issue has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs.

sassanh added kind/bug Something isn't working as expected status/triage This issue needs to be triaged labels Aug 12, 2023

radoering added area/installer Related to the dependency installer and removed status/triage This issue needs to be triaged labels Aug 13, 2023

radoering mentioned this issue Aug 13, 2023

installer: respect source if the same version of a package has been locked from different sources #8304

Merged

2 tasks

radoering closed this as completed in #8304 Aug 17, 2023

github-actions bot locked as resolved and limited conversation to collaborators Feb 29, 2024

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

Environment markers with different sources for a dependency #8303

Environment markers with different sources for a dependency #8303

sassanh commented Aug 12, 2023

sassanh commented Aug 12, 2023

dimbleby commented Aug 13, 2023

sassanh commented Aug 13, 2023

dimbleby commented Aug 13, 2023

sassanh commented Aug 14, 2023

sassanh commented Aug 14, 2023

dimbleby commented Aug 14, 2023 •

edited

sassanh commented Aug 14, 2023

dimbleby commented Aug 14, 2023

sassanh commented Aug 14, 2023

sassanh commented Aug 15, 2023 •

edited

dimbleby commented Aug 15, 2023

sassanh commented Aug 15, 2023

sassanh commented Aug 15, 2023 •

edited

radoering commented Aug 16, 2023

github-actions bot commented Feb 29, 2024

Environment markers with different sources for a dependency #8303

Environment markers with different sources for a dependency #8303

Comments

sassanh commented Aug 12, 2023

Issue

sassanh commented Aug 12, 2023

dimbleby commented Aug 13, 2023

sassanh commented Aug 13, 2023

dimbleby commented Aug 13, 2023

sassanh commented Aug 14, 2023

sassanh commented Aug 14, 2023

dimbleby commented Aug 14, 2023 • edited

sassanh commented Aug 14, 2023

dimbleby commented Aug 14, 2023

sassanh commented Aug 14, 2023

sassanh commented Aug 15, 2023 • edited

dimbleby commented Aug 15, 2023

sassanh commented Aug 15, 2023

sassanh commented Aug 15, 2023 • edited

radoering commented Aug 16, 2023

github-actions bot commented Feb 29, 2024

dimbleby commented Aug 14, 2023 •

edited

sassanh commented Aug 15, 2023 •

edited

sassanh commented Aug 15, 2023 •

edited