Skip to content

Commit

Permalink
Added validation and async for remote_file
Browse files Browse the repository at this point in the history
* Added async download for remote_file for http-archive via block param
* Added validation to check that there is no relative paths present
  • Loading branch information
fzakaria committed Apr 30, 2024
1 parent 345f5c4 commit d29d011
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 13 deletions.
16 changes: 8 additions & 8 deletions src/test/tools/bzlmod/MODULE.bazel.lock
Original file line number Diff line number Diff line change
Expand Up @@ -1105,7 +1105,7 @@
},
"@@bazel_tools//tools/android:android_extensions.bzl%android_sdk_proxy_extensions": {
"general": {
"bzlTransitiveDigest": "HG+Sk8+tt8azZAw+tNthPJivPqU+pTq12U45IPHw4S0=",
"bzlTransitiveDigest": "jJ7dHDwfG+UYsffx62dZlJtqNxeGvVdIh89oAw6uWWE=",
"usagesDigest": "x7fdOxuT/Cd/ZXZa8OyoLFxU34+IE1KtcbASixO3XJM=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
Expand All @@ -1122,7 +1122,7 @@
},
"@@bazel_tools//tools/android:android_extensions.bzl%remote_android_tools_extensions": {
"general": {
"bzlTransitiveDigest": "HG+Sk8+tt8azZAw+tNthPJivPqU+pTq12U45IPHw4S0=",
"bzlTransitiveDigest": "jJ7dHDwfG+UYsffx62dZlJtqNxeGvVdIh89oAw6uWWE=",
"usagesDigest": "LPw+9iUcnRY1k89ZEMEZ/AtOYIK2LgSeKnaiYVCDaag=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
Expand Down Expand Up @@ -1215,7 +1215,7 @@
},
"@@bazel_tools//tools/test:extensions.bzl%remote_coverage_tools_extension": {
"general": {
"bzlTransitiveDigest": "YbrUKgaCsibANLkDI8DEU2uai8UvDsFjd39YDSVoUp8=",
"bzlTransitiveDigest": "5uCJHRgeOdfkIZvSNxWQRC6euKj+/ZXwlg4iwva2TY8=",
"usagesDigest": "O3U7dkKl24l4dKwsK8Y1uf1SAa/eEwLfw4BRsHGugwc=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
Expand Down Expand Up @@ -1280,7 +1280,7 @@
},
"@@rules_java~//java:extensions.bzl%toolchains": {
"general": {
"bzlTransitiveDigest": "1X81l1o+MviLFuKYPXGWVT66BdC7Ty9/EUtRiWS5y4E=",
"bzlTransitiveDigest": "cUjK26QqCL73mEjcZt8RUgY2a43kxbssNdxCXcgxTs4=",
"usagesDigest": "1/Dwy6r0Nf/YUE94OFi5Yy0Mo+TrPxA3U8hBaEzYH5s=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
Expand Down Expand Up @@ -1846,7 +1846,7 @@
},
"@@rules_jvm_external~//:extensions.bzl%maven": {
"general": {
"bzlTransitiveDigest": "9zKradkP0ZQpyk+poE39Yr2NYXaxlmIKCF3+mA6D6Eg=",
"bzlTransitiveDigest": "d2YUqcWGF10XqgvTtsyrUrXieA/yXvDYLDU9E2bCSl4=",
"usagesDigest": "L+U25+BzBiliO0i3XrvqC5up0f210dPakCjqg5MBrp4=",
"recordedFileInputs": {
"@@rules_jvm_external~//rules_jvm_external_deps_install.json": "10442a5ae27d9ff4c2003e5ab71643bf0d8b48dcf968b4173fa274c3232a8c06"
Expand Down Expand Up @@ -2870,7 +2870,7 @@
},
"@@rules_jvm_external~//:non-module-deps.bzl%non_module_deps": {
"general": {
"bzlTransitiveDigest": "XB23DoypqOTVnUqFfxGyCI/7TeD2AbIy1oWjh7czC10=",
"bzlTransitiveDigest": "HvzT2KEX+5VRDFYdhThm5d2vIhO6zdDWM9g+AFKHSZk=",
"usagesDigest": "HWGzpnxaDzDwBkFxYvT/plvwcfNrPZGTg9ZYibwxNGw=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
Expand Down Expand Up @@ -2898,7 +2898,7 @@
},
"@@rules_python~//python/extensions:python.bzl%python": {
"general": {
"bzlTransitiveDigest": "l+fGuO6nOaiH5dPIrKzLKdWsyStDB9tD1p4FQWhGmdo=",
"bzlTransitiveDigest": "scREjR1y1Dw/Jk79jmweL/eruR/qeWuA6ZkchdQFXrk=",
"usagesDigest": "2dLy/xmv7+TD8WRYYIxtxZMeS4nrJZGIDSMkYRE0elw=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
Expand Down Expand Up @@ -2928,7 +2928,7 @@
},
"@@rules_python~//python/extensions/private:internal_deps.bzl%internal_deps": {
"general": {
"bzlTransitiveDigest": "3dG9EuiZuwV5k8fniFn29BZC7E3Fn4b40/5wb/Vspp4=",
"bzlTransitiveDigest": "UkUZZVHg/l6jgAiGe62L/V3VFaXh/0BOVMRyBOAPl8Q=",
"usagesDigest": "MZDuELgGnEk5m0vRt+s02bhPv0B21Oi1jm1KuRqZ5FY=",
"recordedFileInputs": {},
"recordedDirentsInputs": {},
Expand Down
43 changes: 38 additions & 5 deletions tools/build_defs/repo/utils.bzl
Original file line number Diff line number Diff line change
Expand Up @@ -91,6 +91,27 @@ def _download_patch(ctx, patch_url, integrity, auth):
)
return patch_path

def _validate_path(ctx, relative_path):
"""Validate that the path is a child of the repository directory
Effectively tests that the path contains no ".." components.
Args:
ctx: The repository context of the repository rule calling this utility
function.
relative_path: The path to validate.
"""
repo_dir = ctx.path(".")
resolved_repo_dir = repo_dir.realpath
path = repo_dir.get_child(relative_path)
resolved_path = path.realpath

if not str(resolved_path).startswith(str(resolved_repo_dir) + "/"):
fail("Invalid path: '{}' is not a child of the repository directory '{}'."
.format(relative_path, resolved_repo_dir))

return str(path)


def remote_files(ctx, auth = None):
"""Utility function for downloading remote files.
Expand All @@ -103,17 +124,29 @@ def remote_files(ctx, auth = None):
function.
auth: An optional dict specifying authentication information for some of the URLs.
"""
for path, remote_file_urls in ctx.attr.remote_file_urls.items():
# integrity is optional but really recommended
integrity = ctx.attr.remote_file_integrity.get(path)

pending = [
ctx.download(
remote_file_urls,
path,
canonical_id = ctx.attr.canonical_id,
auth = auth,
integrity = integrity,
# integrity is optional but really recommended
integrity = ctx.attr.remote_file_integrity.get(path),
block = False,
)
for path, remote_file_urls in ctx.attr.remote_file_urls.items()]

# Wait until the requess are done
for p in pending:
p.wait()

# kind of silly, but the validation has to happen AFTER
# we download the files.
# The validation relies on `realpath` which throws an error
# if the file does not exist.
for path in ctx.attr.remote_file_urls:
_validate_path(ctx, path)


def patch(ctx, patches = None, patch_cmds = None, patch_cmds_win = None, patch_tool = None, patch_args = None, auth = None):
"""Implementation of patching an already extracted repository.
Expand Down

0 comments on commit d29d011

Please sign in to comment.