From 4f0fb75829a6d5d7c1642dbf1f0b5bec659f6bbd Mon Sep 17 00:00:00 2001 From: George Waters Date: Tue, 29 Nov 2022 17:02:05 -0500 Subject: [PATCH 1/5] Calculate and store hash for url dependencies This performs the same hashing process as is done on file dependencies, for url dependencies. This requires a change in `poetry-core` to work. --- src/poetry/puzzle/provider.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/poetry/puzzle/provider.py b/src/poetry/puzzle/provider.py index 9c7457f7e7c..968d0a8d73b 100644 --- a/src/poetry/puzzle/provider.py +++ b/src/poetry/puzzle/provider.py @@ -431,7 +431,7 @@ def get_package_from_directory(cls, directory: Path) -> Package: return PackageInfo.from_directory(path=directory).to_package(root_dir=directory) def _search_for_url(self, dependency: URLDependency) -> Package: - package = self.get_package_from_url(dependency.url) + package = self.get_package_from_url(dependency.url, dependency.hash) self.validate_package_for_dependency(dependency=dependency, package=package) @@ -446,12 +446,18 @@ def _search_for_url(self, dependency: URLDependency) -> Package: return package @classmethod - def get_package_from_url(cls, url: str) -> Package: + def get_package_from_url( + cls, url: str, hash_func: Callable[..., str] | None = None + ) -> Package: file_name = os.path.basename(urllib.parse.urlparse(url).path) with tempfile.TemporaryDirectory() as temp_dir: dest = Path(temp_dir) / file_name download_file(url, dest) package = cls.get_package_from_file(dest) + if hash_func is not None: + package.files = [ + {"file": file_name, "hash": "sha256:" + hash_func(dest)} + ] package._source_type = "url" package._source_url = url From e7ccd011f5762af9c99c1b59a10c253c75ab6d98 Mon Sep 17 00:00:00 2001 From: George Waters Date: Wed, 30 Nov 2022 11:04:11 -0500 Subject: [PATCH 2/5] Move 'hash' function into utils.helpers --- src/poetry/installation/executor.py | 3 ++- src/poetry/puzzle/provider.py | 20 +++++++++++--------- src/poetry/utils/helpers.py | 11 +++++++++++ 3 files changed, 24 insertions(+), 10 deletions(-) diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index 0a01fee2b98..b3b298caa20 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -28,6 +28,7 @@ from poetry.utils.authenticator import Authenticator from poetry.utils.env import EnvCommandError from poetry.utils.helpers import atomic_open +from poetry.utils.helpers import get_file_hash from poetry.utils.helpers import pluralize from poetry.utils.helpers import remove_directory from poetry.utils.pip import pip_install @@ -667,7 +668,7 @@ def _download_link(self, operation: Install | Update, link: Link) -> Path: @staticmethod def _validate_archive_hash(archive: Path, package: Package) -> str: file_dep = FileDependency(package.name, archive) - archive_hash: str = "sha256:" + file_dep.hash() + archive_hash: str = "sha256:" + get_file_hash(file_dep.full_path) known_hashes = {f["hash"] for f in package.files} if archive_hash not in known_hashes: diff --git a/src/poetry/puzzle/provider.py b/src/poetry/puzzle/provider.py index 968d0a8d73b..cfb50d43b4a 100644 --- a/src/poetry/puzzle/provider.py +++ b/src/poetry/puzzle/provider.py @@ -33,6 +33,7 @@ from poetry.puzzle.exceptions import OverrideNeeded from poetry.repositories.exceptions import PackageNotFound from poetry.utils.helpers import download_file +from poetry.utils.helpers import get_file_hash from poetry.vcs.git import Git @@ -396,7 +397,10 @@ def _search_for_file(self, dependency: FileDependency) -> Package: package.root_dir = dependency.base package.files = [ - {"file": dependency.path.name, "hash": "sha256:" + dependency.hash()} + { + "file": dependency.path.name, + "hash": "sha256:" + get_file_hash(dependency.full_path), + } ] return package @@ -431,7 +435,7 @@ def get_package_from_directory(cls, directory: Path) -> Package: return PackageInfo.from_directory(path=directory).to_package(root_dir=directory) def _search_for_url(self, dependency: URLDependency) -> Package: - package = self.get_package_from_url(dependency.url, dependency.hash) + package = self.get_package_from_url(dependency.url) self.validate_package_for_dependency(dependency=dependency, package=package) @@ -446,18 +450,16 @@ def _search_for_url(self, dependency: URLDependency) -> Package: return package @classmethod - def get_package_from_url( - cls, url: str, hash_func: Callable[..., str] | None = None - ) -> Package: + def get_package_from_url(cls, url: str) -> Package: file_name = os.path.basename(urllib.parse.urlparse(url).path) with tempfile.TemporaryDirectory() as temp_dir: dest = Path(temp_dir) / file_name download_file(url, dest) package = cls.get_package_from_file(dest) - if hash_func is not None: - package.files = [ - {"file": file_name, "hash": "sha256:" + hash_func(dest)} - ] + + package.files = [ + {"file": file_name, "hash": "sha256:" + get_file_hash(dest)} + ] package._source_type = "url" package._source_url = url diff --git a/src/poetry/utils/helpers.py b/src/poetry/utils/helpers.py index c4ef660d547..4d6ff50e28a 100644 --- a/src/poetry/utils/helpers.py +++ b/src/poetry/utils/helpers.py @@ -1,5 +1,7 @@ from __future__ import annotations +import hashlib +import io import os import shutil import stat @@ -252,3 +254,12 @@ def get_real_windows_path(path: str | Path) -> Path: path = path.resolve() return path + + +def get_file_hash(path: Path, hash_name: str = "sha256") -> str: + h = hashlib.new(hash_name) + with path.open("rb") as fp: + for content in iter(lambda: fp.read(io.DEFAULT_BUFFER_SIZE), b""): + h.update(content) + + return h.hexdigest() From 778c175a8b554c0e312d5c9fbb90710c00de10ee Mon Sep 17 00:00:00 2001 From: George Waters Date: Wed, 30 Nov 2022 15:30:54 -0500 Subject: [PATCH 3/5] Add file and hash data to url file fixtures --- .../fixtures/with-same-version-url-dependencies.test | 9 ++++++--- tests/installation/fixtures/with-url-dependency.test | 4 +++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/tests/installation/fixtures/with-same-version-url-dependencies.test b/tests/installation/fixtures/with-same-version-url-dependencies.test index e80c72a3d03..bc509936da6 100644 --- a/tests/installation/fixtures/with-same-version-url-dependencies.test +++ b/tests/installation/fixtures/with-same-version-url-dependencies.test @@ -5,7 +5,9 @@ description = "" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [] +files = [ + {file = "demo-0.1.0-py2.py3-none-any.whl", hash = "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"} +] [package.source] type = "url" @@ -25,8 +27,9 @@ description = "" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [] - +files = [ + {file = "demo-0.1.0.tar.gz", hash = "sha256:72e8531e49038c5f9c4a837b088bfcb8011f4a9f76335c8f0654df6ac539b3d6"} +] [package.source] type = "url" url = "https://python-poetry.org/distributions/demo-0.1.0.tar.gz" diff --git a/tests/installation/fixtures/with-url-dependency.test b/tests/installation/fixtures/with-url-dependency.test index 13cab35bc46..e6878546942 100644 --- a/tests/installation/fixtures/with-url-dependency.test +++ b/tests/installation/fixtures/with-url-dependency.test @@ -5,7 +5,9 @@ description = "" category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" -files = [] +files = [ + {file = "demo-0.1.0-py2.py3-none-any.whl", hash = "sha256:70e704135718fffbcbf61ed1fc45933cfd86951a744b681000eaaa75da31f17a"} +] [package.source] type = "url" From af1687c2984a4ff541529985f282f7b93cfb747a Mon Sep 17 00:00:00 2001 From: George Waters Date: Wed, 30 Nov 2022 16:34:44 -0500 Subject: [PATCH 4/5] Add tests for 'get_file_hash' These tests were copied over from 'poetry-core' and updated to work here. --- tests/utils/test_helpers.py | 76 +++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) diff --git a/tests/utils/test_helpers.py b/tests/utils/test_helpers.py index b5ab3cfebdc..835298fa405 100644 --- a/tests/utils/test_helpers.py +++ b/tests/utils/test_helpers.py @@ -1,7 +1,18 @@ from __future__ import annotations +from pathlib import Path +from typing import TYPE_CHECKING + +import pytest + from poetry.core.utils.helpers import parse_requires +from poetry.utils.helpers import get_file_hash + + +if TYPE_CHECKING: + from tests.types import FixtureDirGetter + def test_parse_requires(): requires = """\ @@ -57,3 +68,68 @@ def test_parse_requires(): ] # fmt: on assert result == expected + + +def test_default_hash(fixture_dir: FixtureDirGetter) -> None: + root_dir = Path(__file__).parent.parent.parent + file_path = root_dir / fixture_dir("distributions/demo-0.1.0.tar.gz") + sha_256 = "72e8531e49038c5f9c4a837b088bfcb8011f4a9f76335c8f0654df6ac539b3d6" + assert get_file_hash(file_path) == sha_256 + + +try: + from hashlib import algorithms_guaranteed +except ImportError: + algorithms_guaranteed = {"md5", "sha1", "sha224", "sha256", "sha384", "sha512"} + + +@pytest.mark.parametrize( + "hash_name,expected", + [ + (hash_name, value) + for hash_name, value in [ + ("sha224", "972d02f36539a98599aed0566bc8aaf3e6701f4e895dd797d8f5248e"), + ( + "sha3_512", + "c04ee109ae52d6440445e24dbd6d244a1d0f0289ef79cb7ba9bc3c139c0237169af9a8f61cd1cf4fc17f853ddf84f97c475ac5bb6c91a4aff0b825b884d4896c", # noqa: E501 + ), + ( + "blake2s", + "c336ecbc9d867c9d860accfba4c3723c51c4b5c47a1e0a955e1c8df499e36741", + ), + ( + "sha3_384", + "d4abb2459941369aabf8880c5287b7eeb80678e14f13c71b9ecf64c772029dc3f93939590bea9ecdb51a1d1a74fefc5a", # noqa: E501 + ), + ( + "blake2b", + "48e70abac547ab38e2330e6e6743a0c0f6274dcaa6df2c98135a78a9dd5b04a072d551fc3851b34da03eb0bf50dd71c7f32a8c36956e99fd6c66491bc7844800", # noqa: E501 + ), + ( + "sha256", + "72e8531e49038c5f9c4a837b088bfcb8011f4a9f76335c8f0654df6ac539b3d6", + ), + ( + "sha512", + "e08a00a4b86358e49a318e7e3ba7a3d2fabdd17a2fef95559a0af681ea07ab1296b0b8e11e645297da296290661dc07ae3c8f74eab66bd18a80dce0c0ccb355b", # noqa: E501 + ), + ( + "sha384", + "aa3144e28c6700a83247e8ec8711af5d3f5f75997990d48ec41e66bd275b3d0e19ee6f2fe525a358f874aa717afd06a9", # noqa: E501 + ), + ("sha3_224", "64bfc6e4125b4c6d67fd88ad1c7d1b5c4dc11a1970e433cd576c91d4"), + ("sha1", "4c057579005ac3e68e951a11ffdc4b27c6ae16af"), + ( + "sha3_256", + "ba3d2a964b0680b6dc9565a03952e29c294c785d5a2307d3e2d785d73b75ed7e", + ), + ] + if hash_name in algorithms_guaranteed + ], +) +def test_guaranteed_hash( + hash_name: str, expected: str, fixture_dir: FixtureDirGetter +) -> None: + root_dir = Path(__file__).parent.parent.parent + file_path = root_dir / fixture_dir("distributions/demo-0.1.0.tar.gz") + assert get_file_hash(file_path, hash_name) == expected From 3858bf09280a9f2f27e56cb4b557b38ff72fbb52 Mon Sep 17 00:00:00 2001 From: George Waters Date: Wed, 7 Dec 2022 10:33:41 -0500 Subject: [PATCH 5/5] Don't use FileDependency in executor Don't use FileDependency when validating the archive hash in executor, just compute the hash with the archive path directly. --- src/poetry/installation/executor.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/poetry/installation/executor.py b/src/poetry/installation/executor.py index b3b298caa20..6df976f7c45 100644 --- a/src/poetry/installation/executor.py +++ b/src/poetry/installation/executor.py @@ -15,7 +15,6 @@ from typing import Any from cleo.io.null_io import NullIO -from poetry.core.packages.file_dependency import FileDependency from poetry.core.packages.utils.link import Link from poetry.core.pyproject.toml import PyProjectTOML @@ -667,8 +666,7 @@ def _download_link(self, operation: Install | Update, link: Link) -> Path: @staticmethod def _validate_archive_hash(archive: Path, package: Package) -> str: - file_dep = FileDependency(package.name, archive) - archive_hash: str = "sha256:" + get_file_hash(file_dep.full_path) + archive_hash: str = "sha256:" + get_file_hash(archive) known_hashes = {f["hash"] for f in package.files} if archive_hash not in known_hashes: