feat(builtin): introduce copy_to_bin rule (#1450)

* feat(builtin): introduce copy_to_bin rule

Based on meteorcloudy@ awesome work in bazelbuild/bazel-skylib#217
That PR isn't getting approved for upstream so we vendor it into our own ruleset

* chore: migrate to copy_file and copy_to_bin

examples/kotlin/BUILD.bazel

@@ -1,7 +1,7 @@
 # Add rules here to build your software
 # See https://docs.bazel.build/versions/master/build-ref.html#BUILD_files
 
-load("@build_bazel_rules_nodejs//:index.bzl", "pkg_web")
+load("@build_bazel_rules_nodejs//:index.bzl", "copy_to_bin", "pkg_web")
 load("@io_bazel_rules_kotlin//kotlin:kotlin.bzl", "kt_js_import", "kt_js_library")
 load("@npm//http-server:index.bzl", "http_server")
 load("@npm_bazel_jasmine//:index.bzl", "jasmine_node_test")
@@ -22,20 +22,19 @@ kt_js_library(
     deps = [":kotlinx-html-js"],
 )
 
-# Copy bootstrap.js to the bin folder as _bootstrap.js
-# so that relative import to `./hello.js` is valid
-genrule(
+# Copy bootstrap.js to the output folder, so all files are next to each other at runtime
+# Allows the `./hello.js` relative import to work while referencing an output file
+copy_to_bin(
     name = "bootstrap",
     srcs = ["bootstrap.js"],
-    outs = ["_bootstrap.js"],
-    cmd = "cp $< $@",
 )
 
 rollup_bundle(
     name = "bundle",
     srcs = ["hello.js"],
     config_file = "rollup.config.js",
-    entry_point = "_bootstrap.js",
+    # Reference the copy of bootstrap.js in the output folder
+    entry_point = "bootstrap",
     # TODO: make this example work with format = "esm"
     format = "cjs",
     output_dir = True,

index.bzl

@@ -19,6 +19,7 @@ Users should not load files under "/internal"
 
 load("//internal/common:check_bazel_version.bzl", _check_bazel_version = "check_bazel_version")
 load("//internal/common:check_version.bzl", "check_version")
+load("//internal/common:copy_to_bin.bzl", _copy_to_bin = "copy_to_bin")
 load("//internal/jasmine_node_test:jasmine_node_test.bzl", _jasmine_node_test = "jasmine_node_test")
 load(
     "//internal/node:node.bzl",
@@ -39,6 +40,7 @@ jasmine_node_test = _jasmine_node_test
 npm_package = _npm_package
 npm_package_bin = _npm_bin
 pkg_web = _pkg_web
+copy_to_bin = _copy_to_bin
 # ANY RULES ADDED HERE SHOULD BE DOCUMENTED, see index.for_docs.bzl
 
 # Allows us to avoid a transitive dependency on bazel_skylib from leaking to users

index.for_docs.bzl

@@ -17,6 +17,7 @@
 This differs from :index.bzl because we don't have wrapping macros that hide the real doc"""
 
 load("//internal/common:check_bazel_version.bzl", _check_bazel_version = "check_bazel_version")
+load("//internal/common:copy_to_bin.bzl", _copy_to_bin = "copy_to_bin")
 load("//internal/node:node.bzl", _nodejs_binary = "nodejs_binary", _nodejs_test = "nodejs_test")
 load("//internal/node:node_repositories.bzl", _node_repositories = "node_repositories_rule")
 load("//internal/node:npm_package_bin.bzl", _npm_bin = "npm_package_bin")
@@ -25,6 +26,7 @@ load("//internal/npm_package:npm_package.bzl", _npm_package = "npm_package")
 load("//internal/pkg_web:pkg_web.bzl", _pkg_web = "pkg_web")
 
 check_bazel_version = _check_bazel_version
+copy_to_bin = _copy_to_bin
 nodejs_binary = _nodejs_binary
 nodejs_test = _nodejs_test
 node_repositories = _node_repositories

internal/common/BUILD.bazel

@@ -21,7 +21,9 @@ package(default_visibility = ["//internal:__subpackages__"])
 
 bzl_library(
     name = "bzl",
-    srcs = glob(["*.bzl"]),
+    srcs = glob(["*.bzl"]) + [
+        "//third_party/github.com/bazelbuild/bazel-skylib:bzl",
+    ],
     visibility = ["//visibility:public"],
 )
 

internal/common/copy_to_bin.bzl

@@ -0,0 +1,65 @@
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"copy_to_bin() rule"
+
+load("//third_party/github.com/bazelbuild/bazel-skylib:rules/private/copy_file_private.bzl", "copy_bash", "copy_cmd")
+
+def _copy_to_bin_impl(ctx):
+    all_dst = []
+    for src in ctx.files.srcs:
+        if not src.is_source:
+            fail("A source file must be specified in copy_to_bin rule, %s is not a source file." % src.path)
+        dst = ctx.actions.declare_file(src.basename, sibling = src)
+        if ctx.attr.is_windows:
+            copy_cmd(ctx, src, dst)
+        else:
+            copy_bash(ctx, src, dst)
+        all_dst.append(dst)
+    return DefaultInfo(files = depset(all_dst), runfiles = ctx.runfiles(files = all_dst))
+
+_copy_to_bin = rule(
+    implementation = _copy_to_bin_impl,
+    attrs = {
+        "srcs": attr.label_list(mandatory = True, allow_files = True),
+        "is_windows": attr.bool(mandatory = True, doc = "Automatically set by macro"),
+    },
+)
+
+def copy_to_bin(name, srcs, **kwargs):
+    """Copies a source file to bazel-bin at the same workspace-relative path path.
+
+    e.g. `<workspace_root>/foo/bar/a.txt -> <bazel-bin>/foo/bar/a.txt`
+
+    This is useful to populate the output folder with all files needed at runtime, even
+    those which aren't outputs of a Bazel rule.
+
+    This way you can run a binary in the output folder (execroot or runfiles_root)
+    without that program needing to rely on a runfiles helper library or be aware that
+    files are divided between the source tree and the output tree.
+
+    Args:
+        name: Name of the rule.
+        srcs: A List of Labels. File(s) to to copy.
+        **kwargs: further keyword arguments, e.g. `visibility`
+    """
+    _copy_to_bin(
+        name = name,
+        srcs = srcs,
+        is_windows = select({
+            "@bazel_tools//src/conditions:host_windows": True,
+            "//conditions:default": False,
+        }),
+        **kwargs
+    )

internal/common/test/BUILD.bazel

@@ -0,0 +1,20 @@
+load("//internal/common:copy_to_bin.bzl", "copy_to_bin")
+
+licenses(["notice"])
+
+package(default_testonly = 1)
+
+sh_test(
+    name = "copy_to_bin_tests",
+    srcs = ["copy_to_bin_tests.sh"],
+    data = [
+        ":a",
+        "//third_party/github.com/bazelbuild/bazel-skylib:tests/unittest.bash",
+    ],
+    deps = ["@bazel_tools//tools/bash/runfiles"],
+)
+
+copy_to_bin(
+    name = "a",
+    srcs = ["foo/bar/a.txt"],
+)

internal/common/test/copy_to_bin_tests.sh

@@ -0,0 +1,51 @@
+# Copyright 2019 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# --- begin runfiles.bash initialization ---
+# Copy-pasted from Bazel's Bash runfiles library (tools/bash/runfiles/runfiles.bash).
+set -euo pipefail
+if [[ ! -d "${RUNFILES_DIR:-/dev/null}" && ! -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  if [[ -f "$0.runfiles_manifest" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles_manifest"
+  elif [[ -f "$0.runfiles/MANIFEST" ]]; then
+    export RUNFILES_MANIFEST_FILE="$0.runfiles/MANIFEST"
+  elif [[ -f "$0.runfiles/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+    export RUNFILES_DIR="$0.runfiles"
+  fi
+fi
+if [[ -f "${RUNFILES_DIR:-/dev/null}/bazel_tools/tools/bash/runfiles/runfiles.bash" ]]; then
+  source "${RUNFILES_DIR}/bazel_tools/tools/bash/runfiles/runfiles.bash"
+elif [[ -f "${RUNFILES_MANIFEST_FILE:-/dev/null}" ]]; then
+  source "$(grep -m1 "^bazel_tools/tools/bash/runfiles/runfiles.bash " \
+            "$RUNFILES_MANIFEST_FILE" | cut -d ' ' -f 2-)"
+else
+  echo >&2 "ERROR: cannot find @bazel_tools//tools/bash/runfiles:runfiles.bash"
+  exit 1
+fi
+# --- end runfiles.bash initialization ---
+
+source "$(rlocation build_bazel_rules_nodejs/third_party/github.com/bazelbuild/bazel-skylib/tests/unittest.bash)" \
+  || { echo "Could not source build_bazel_rules_nodejs/third_party/github.com/bazelbuild/bazel-skylib/tests/unittest.bash" >&2; exit 1; }
+
+function test_map_to_output() {
+  echo "$(rlocation build_bazel_rules_nodejs/internal/common/test/foo/bar/a.txt)" >"$TEST_log"
+  # Test the foo/bar/a.txt is copied to bazel-out/
+  expect_log 'bazel-out/'
+  cat "$(rlocation build_bazel_rules_nodejs/internal/common/test/foo/bar/a.txt)" >"$TEST_log"
+  # Test the content of foo/bar/a.txt is correct
+  expect_log '#!/bin/bash'
+  expect_log '^echo aaa$'
+}
+
+run_suite "map_to_output test suite"

internal/common/test/foo/bar/a.txt

@@ -0,0 +1,2 @@
+#!/bin/bash
+echo aaa

internal/linker/test/BUILD.bazel

@@ -15,6 +15,6 @@ jasmine_node_test(
     name = "unit_tests",
     srcs = ["test_lib"],
     # NB: for better dev round-trip, we test against the ts_library target
-    # rather than update the index.js it's transpiled from.
+    # rather than update the index.js it's transpiled to.
     data = ["//internal/linker:linker_lib"],
 )

internal/linker/test/integration/BUILD.bazel

@@ -33,7 +33,8 @@ linked(
     out = "actual",
     program = ":run_program",
     deps = [
-        "//%s/absolute_import:index.js" % package_name(),
+        # NB: reference the copy of index.js in the output folder
+        "//%s/absolute_import:copy_to_bin" % package_name(),
         ":run_program",
         "//internal/linker/test/integration/dynamic_linked_pkg",
         "//internal/linker/test/integration/dynamic_linked_scoped_pkg",

internal/linker/test/integration/absolute_import/BUILD.bazel

@@ -1,8 +1,8 @@
 package(default_visibility = ["//internal/linker/test:__subpackages__"])
 
-genrule(
+load("//:index.bzl", "copy_to_bin")
+
+copy_to_bin(
     name = "copy_to_bin",
-    srcs = ["index.js_"],
-    outs = ["index.js"],
-    cmd = "cp $< $@",
+    srcs = ["index.js"],
 )

packages/create/BUILD.bazel

@@ -1,11 +1,11 @@
 load("@build_bazel_rules_nodejs//:tools/defaults.bzl", "nodejs_test", "npm_package")
+load("//third_party/github.com/bazelbuild/bazel-skylib:rules/copy_file.bzl", "copy_file")
 
 # Copy common bazelrc file to be included in this package
-genrule(
+copy_file(
     name = "copy_bazelrc",
-    srcs = ["//:common.bazelrc"],
-    outs = ["common.bazelrc"],
-    cmd = "cp $< $@",
+    src = "//:common.bazelrc",
+    out = "common.bazelrc",
 )
 
 npm_package(

packages/rollup/test/integration/foo/BUILD.bazel

@@ -1,16 +1,12 @@
 load("@npm_bazel_typescript//:index.from_src.bzl", "ts_library")
+load("//third_party/github.com/bazelbuild/bazel-skylib:rules/copy_file.bzl", "copy_file")
 
 package(default_visibility = ["//packages/rollup:__subpackages__"])
 
-genrule(
+copy_file(
     name = "copy_user",
-    srcs = [
-        ":user.mjs",
-    ],
-    outs = [
-        ":user.js",
-    ],
-    cmd = "cp $< $@",
+    src = ":user.mjs",
+    out = ":user.js",
 )
 
 ts_library(

packages/worker/BUILD.bazel

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 load("@build_bazel_rules_nodejs//:tools/defaults.bzl", "npm_package")
+load("//third_party/github.com/bazelbuild/bazel-skylib:rules/copy_file.bzl", "copy_file")
 
 # We reach inside the @bazel/typescript npm package to grab this one .js file
 # This avoids a complex refactoring where we extract that .ts file from tsc_wrapped to a common library
@@ -22,6 +23,8 @@ _worker_path = "external/build_bazel_rules_typescript/internal/tsc_wrapped/worke
 # so the runtime require() statements still work
 _worker_proto_dir = "third_party/github.com/bazelbuild/bazel/src/main/protobuf"
 
+# because the package is a tree artifact, there's no label to address the files we want
+# and so we must use a custom cp command, not the copy_file rule
 genrule(
     name = "copy_worker_js",
     srcs = ["@build_bazel_rules_typescript//:npm_bazel_typescript_package"],
@@ -36,11 +39,10 @@ genrule(
     cmd = "cp $(location @build_bazel_rules_typescript//:npm_bazel_typescript_package)/%s.d.ts $@" % _worker_path,
 )
 
-genrule(
+copy_file(
     name = "copy_worker_proto",
-    srcs = ["@build_bazel_rules_typescript//%s:worker_protocol.proto" % _worker_proto_dir],
-    outs = ["%s/worker_protocol.proto" % _worker_proto_dir],
-    cmd = "cp $< $@",
+    src = "@build_bazel_rules_typescript//%s:worker_protocol.proto" % _worker_proto_dir,
+    out = "%s/worker_protocol.proto" % _worker_proto_dir,
 )
 
 npm_package(

third_party/github.com/bazelbuild/bazel-skylib/BUILD

@@ -2,15 +2,20 @@ load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 
 licenses(["notice"])
 
+exports_files(["tests/unittest.bash"])
+
 filegroup(
     name = "package_contents",
-    srcs = glob(["**"]),
+    srcs = glob(["**"], exclude = ["tests/**"]),
     visibility = ["//:__pkg__"],
 )
 
 bzl_library(
     name = "bzl",
-    srcs = glob(["lib/*.bzl"]),
+    srcs = glob([
+        "lib/*.bzl",
+        "rules/**/*.bzl",
+    ]),
     visibility = ["//visibility:public"],
     deps = [
         "//internal/npm_install:bzl",