Add support for remote_files for http_archive

This issue adds support necessary to tackle bazelbuild/bazel-central-registry#1566 Add two new attributes to http_archive: remote_file_urls and remote_file_integrity. The purpose of these two attributes is to allow files to effectively be overlaid ontop of an http_archive. The goal of such functionality would be useful for BCR since the BUILD & WORKSPACE files need no longer be stored as patch files. This means we could probably deprecate `build_file` since that could be referenced as a file:// url in the remote_file_urls attribute. Co-authored-by: Mark Williams <markrwilliams@google.com> CC @fmeum Closes bazelbuild#22155. PiperOrigin-RevId: 632594203 Change-Id: I6310093482c5c58537ed6dbe4ff90bafdbd696ff
Kila2 · May 13, 2024 · eda0eea · eda0eea
1 parent 39438e3
commit eda0eea
Show file tree

Hide file tree

Showing 6 changed files with 394 additions and 12 deletions.
diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock
@@ -2917,7 +2917,7 @@
         "bzlTransitiveDigest": "tunTSmgwd2uvTzkCLtdbuCp0AI+WR+ftiPNqZ0rmcZk=",
         "recordedFileInputs": {
           "@@//MODULE.bazel": "939a48e4dbb71472db80f16a6a639f65cb4ddb07d8e1fd01db14cf1d0323917d",
-          "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "1fffc063b7830519a4f8ea2b8a1192fef8c4065e4f2e5a1aeedd4f563a5bc683"
+          "@@//src/test/tools/bzlmod/MODULE.bazel.lock": "43259b270347fc1a76734df4b73b2d8c4e2c14293f3c0fb7dca5c48fd4b22758"
         },
         "recordedDirentsInputs": {},
         "envVariables": {},
@@ -3469,7 +3469,7 @@
     },
     "//tools/android:android_extensions.bzl%android_sdk_proxy_extensions": {
       "general": {
-        "bzlTransitiveDigest": "JQoCqdz3xQGJ+/ILJLx/pnLPCjAYRvq3Tmj3wkpmflI=",
+        "bzlTransitiveDigest": "XLVk7GLje5iUTApmiCu5fapV2+/7HqWPF7cQEu7SPOE=",
         "recordedFileInputs": {},
         "recordedDirentsInputs": {},
         "envVariables": {},
@@ -3485,7 +3485,7 @@
     },
     "//tools/android:android_extensions.bzl%remote_android_tools_extensions": {
       "general": {
-        "bzlTransitiveDigest": "JQoCqdz3xQGJ+/ILJLx/pnLPCjAYRvq3Tmj3wkpmflI=",
+        "bzlTransitiveDigest": "XLVk7GLje5iUTApmiCu5fapV2+/7HqWPF7cQEu7SPOE=",
         "recordedFileInputs": {},
         "recordedDirentsInputs": {},
         "envVariables": {},
@@ -3512,7 +3512,7 @@
     },
     "//tools/test:extensions.bzl%remote_coverage_tools_extension": {
       "general": {
-        "bzlTransitiveDigest": "kcwMubtgxBru6ttbkwo5IjUmLeSz0AODaQoox5L4zVU=",
+        "bzlTransitiveDigest": "ia4DMKKuTJ4QAHKSrRs4v/QBfK48p52gn2SwHL6BXd8=",
         "recordedFileInputs": {},
         "recordedDirentsInputs": {},
         "envVariables": {},

diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD
@@ -803,6 +803,17 @@ sh_test(
     shard_count = 3,
 )
 
+sh_test(
+    name = "external_remote_file_test",
+    size = "small",
+    srcs = ["external_remote_file_test.sh"],
+    data = [
+        ":test-deps",
+        "@bazel_tools//tools/bash/runfiles",
+    ],
+    shard_count = 3,
+)
+
 sh_test(
     name = "external_path_test",
     size = "medium",

diff --git a/src/test/shell/bazel/external_remote_file_test.sh b/src/test/shell/bazel/external_remote_file_test.sh
@@ -0,0 +1,323 @@
+#!/bin/bash
+#
+# Copyright 2017 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Tests the patching functionality of external repositories.
+
+set -euo pipefail
+# --- begin runfiles.bash initialization ---
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  if [[ -f "$0.runfiles_manifest" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+    export RUNFILES_DIR="$0.runfiles"
+  fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+  exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+source "$(rlocation "io_bazel/src/test/shell/integration_test_setup.sh")" \
+  || { echo "integration_test_setup.sh not found!" >&2; exit 1; }
+
+# `uname` returns the current platform, e.g "MSYS_NT-10.0" or "Linux".
+# `tr` converts all upper case letters to lower case.
+# `case` matches the result if the `uname | tr` expression to string prefixes
+# that use the same wildcards as names do in Bash, i.e. "msys*" matches strings
+# starting with "msys", and "*" matches everything (it's the default case).
+case "$(uname -s | tr [:upper:] [:lower:])" in
+msys*)
+  # As of 2019-01-15, Bazel on Windows only supports MSYS Bash.
+  declare -r is_windows=true
+  ;;
+*)
+  declare -r is_windows=false
+  ;;
+esac
+
+if "$is_windows"; then
+  # Disable MSYS path conversion that converts path-looking command arguments to
+  # Windows paths (even if they arguments are not in fact paths).
+  export MSYS_NO_PATHCONV=1
+  export MSYS2_ARG_CONV_EXCL="*"
+fi
+
+
+if $is_windows; then
+  export MSYS_NO_PATHCONV=1
+  export MSYS2_ARG_CONV_EXCL="*"
+fi
+
+function set_up() {
+  WRKDIR=$(mktemp -d "${TEST_TMPDIR}/testXXXXXX")
+  cd "${WRKDIR}"
+  write_default_lockfile "MODULE.bazel.lock"
+  # create an archive file with files interesting for patching
+  mkdir hello_world-0.1.2
+  cat > hello_world-0.1.2/hello_world.c <<'EOF'
+#include <stdio.h>
+int main() {
+  printf("Hello, world!\n");
+  return 0;
+}
+EOF
+  zip hello_world.zip hello_world-0.1.2/*
+  rm -rf hello_world-0.1.2
+}
+
+function get_extrepourl() {
+  if $is_windows; then
+    echo "file:///$(cygpath -m $1)"
+  else
+    echo "file://$1"
+  fi
+}
+
+
+test_overlay_remote_file_multiple_segments() {
+  EXTREPODIR=`pwd`
+  EXTREPOURL="$(get_extrepourl ${EXTREPODIR})"
+
+  # Generate the remote files to overlay
+  mkdir -p child
+    cat > child/foo_bar.c <<'EOF'
+#include <stdio.h>
+int main() {
+  printf("Foo, Bar!\n");
+  return 0;
+}
+EOF
+  cat > child/BUILD.bazel <<'EOF'
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
+cc_binary(
+    name = "foo_bar",
+    srcs = ["foo_bar.c"],
+)
+EOF
+  touch BUILD.bazel
+  touch WORKSPACE
+
+  mkdir main
+  cd main
+  cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+  name="hello_world",
+  strip_prefix="hello_world-0.1.2",
+  urls=["${EXTREPOURL}/hello_world.zip"],
+  remote_file_urls={
+    "WORKSPACE": ["${EXTREPOURL}/WORKSPACE"],
+    "BUILD.bazel": ["${EXTREPOURL}/BUILD.bazel"],
+    "child/foo_bar.c": ["${EXTREPOURL}/child/foo_bar.c"],
+    "child/BUILD.bazel": ["${EXTREPOURL}/child/BUILD.bazel"],
+  },
+)
+EOF
+  write_default_lockfile "MODULE.bazel.lock"
+
+  bazel build @hello_world//child:foo_bar
+}
+
+test_overlay_remote_file_with_integrity() {
+  EXTREPODIR=`pwd`
+  EXTREPOURL="$(get_extrepourl ${EXTREPODIR})"
+
+  # Generate the remote files to overlay
+  cat > BUILD.bazel <<'EOF'
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
+cc_binary(
+    name = "hello_world",
+    srcs = ["hello_world.c"],
+)
+EOF
+  touch WORKSPACE
+
+  mkdir main
+  cd main
+  cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+  name="hello_world",
+  strip_prefix="hello_world-0.1.2",
+  urls=["${EXTREPOURL}/hello_world.zip"],
+  remote_file_urls={
+    "WORKSPACE": ["${EXTREPOURL}/WORKSPACE"],
+    "BUILD.bazel": ["${EXTREPOURL}/BUILD.bazel"],
+  },
+  remote_file_integrity={
+    "WORKSPACE": "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=",
+    "BUILD.bazel": "sha256-0bs+dwSOzHTbNAgDS02I3giLAZu2/NLn7BJWwQGN/Pk=",
+  },
+)
+EOF
+  write_default_lockfile "MODULE.bazel.lock"
+
+  bazel build @hello_world//:hello_world
+}
+
+test_overlay_remote_file_fail_with_integrity() {
+  EXTREPODIR=`pwd`
+  EXTREPOURL="$(get_extrepourl ${EXTREPODIR})"
+
+  # Generate the remote files to overlay
+  cat > BUILD.bazel <<'EOF'
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
+cc_binary(
+    name = "hello_world",
+    srcs = ["hello_world.c"],
+)
+EOF
+  touch WORKSPACE
+
+  mkdir main
+  cd main
+  cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+  name="hello_world",
+  strip_prefix="hello_world-0.1.2",
+  urls=["${EXTREPOURL}/hello_world.zip"],
+  remote_file_urls={
+    "WORKSPACE": ["${EXTREPOURL}/WORKSPACE"],
+    "BUILD.bazel": ["${EXTREPOURL}/BUILD.bazel"],
+  },
+  remote_file_integrity={
+    "WORKSPACE": "sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFZ=",
+    "BUILD.bazel": "sha256-0bs+dwSOzHTbNAgDS02I3giLAZu2/NLn7BJWwQGN/Pk=",
+  },
+)
+EOF
+  write_default_lockfile "MODULE.bazel.lock"
+
+  bazel build @hello_world//:hello_world &> $TEST_log 2>&1 && fail "Expected to fail"
+  expect_log "but wanted sha256-47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFY="
+}
+
+test_overlay_remote_file_without_integrity() {
+  EXTREPODIR=`pwd`
+  EXTREPOURL="$(get_extrepourl ${EXTREPODIR})"
+
+  # Generate the remote files to overlay
+  cat > BUILD.bazel <<'EOF'
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
+cc_binary(
+    name = "hello_world",
+    srcs = ["hello_world.c"],
+)
+EOF
+  touch WORKSPACE
+
+  mkdir main
+  cd main
+  cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+  name="hello_world",
+  strip_prefix="hello_world-0.1.2",
+  urls=["${EXTREPOURL}/hello_world.zip"],
+  remote_file_urls={
+    "WORKSPACE": ["${EXTREPOURL}/WORKSPACE"],
+    "BUILD.bazel": ["${EXTREPOURL}/BUILD.bazel"],
+  },
+)
+EOF
+  write_default_lockfile "MODULE.bazel.lock"
+
+  bazel build @hello_world//:hello_world
+}
+
+test_overlay_remote_file_disallow_relative_outside_repo() {
+  EXTREPODIR=`pwd`
+  EXTREPOURL="$(get_extrepourl ${EXTREPODIR})"
+
+  # Generate the remote files to overlay
+  cat > BUILD.bazel <<'EOF'
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
+cc_binary(
+    name = "hello_world",
+    srcs = ["hello_world.c"],
+)
+EOF
+  touch WORKSPACE
+
+  mkdir main
+  cd main
+  cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+  name="hello_world",
+  strip_prefix="hello_world-0.1.2",
+  urls=["${EXTREPOURL}/hello_world.zip"],
+  remote_file_urls={
+    "../../../WORKSPACE": ["${EXTREPOURL}/WORKSPACE"],
+    "BUILD.bazel": ["${EXTREPOURL}/BUILD.bazel"],
+  },
+)
+EOF
+  write_default_lockfile "MODULE.bazel.lock"
+
+  bazel build @hello_world//:hello_world &> $TEST_log 2>&1 && fail "Expected to fail"
+  expect_log "Error in download: Cannot write outside of the repository directory"
+}
+
+test_overlay_remote_file_disallow_absolute_path() {
+  EXTREPODIR=`pwd`
+  EXTREPOURL="$(get_extrepourl ${EXTREPODIR})"
+
+  cat > BUILD.bazel <<'EOF'
+load("@rules_cc//cc:defs.bzl", "cc_binary")
+
+cc_binary(
+    name = "hello_world",
+    srcs = ["hello_world.c"],
+)
+EOF
+  touch WORKSPACE
+
+  mkdir main
+  cd main
+  cat > WORKSPACE <<EOF
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+http_archive(
+  name="hello_world",
+  strip_prefix="hello_world-0.1.2",
+  urls=["${EXTREPOURL}/hello_world.zip"],
+  remote_file_urls={
+    "/tmp/WORKSPACE": ["${EXTREPOURL}/WORKSPACE"],
+    "BUILD.bazel": ["${EXTREPOURL}/BUILD.bazel"],
+  },
+)
+EOF
+  write_default_lockfile "MODULE.bazel.lock"
+
+  bazel build @hello_world//:hello_world &> $TEST_log 2>&1 && fail "Expected to fail"
+  expect_log "Error in download: Cannot write outside of the repository directory"
+}
+
+run_suite "external remote file tests"