feat(tar): add ergonomic way to strip_prefix (#829)

aspect-build · May 1, 2024 · 977f27f · 977f27f
1 parent c0607f6
commit 977f27f
Show file tree

Hide file tree

Showing 8 changed files with 155 additions and 41 deletions.
diff --git a/docs/tar.md b/docs/tar.md
diff --git a/lib/private/BUILD.bazel b/lib/private/BUILD.bazel
@@ -5,6 +5,7 @@ exports_files(
     [
         "diff_test_tmpl.sh",
         "diff_test_tmpl.bat",
+        "modify_mtree.awk",
         "parse_status_file.jq",
         "parse_status_file.yq",
     ],

diff --git a/lib/private/modify_mtree.awk b/lib/private/modify_mtree.awk
@@ -0,0 +1,32 @@
+# Edits mtree files. See the modify_mtree macro in /lib/tar.bzl.
+{
+    if (strip_prefix != "") {
+        if ($1 == strip_prefix) {
+            # this line declares the directory which is now the root. It may be discarded.
+            next;
+        } else if (index($1, strip_prefix) == 1) {
+            # this line starts with the strip_prefix
+            sub("^" strip_prefix "/", "");
+        } else {
+            # this line declares some path under a parent directory, which will be discarded
+            next;
+        }
+    }
+
+    if (mtime != "") {
+        sub(/time=[0-9\.]+/, "time=" mtime);
+    }
+
+    if (owner != "") {
+        sub(/uid=[0-9\.]+/, "uid=" owner)
+    }
+
+    if (ownername != "") {
+        sub(/uname=[^ ]+/, "uname=" ownername)
+    }
+
+    if (package_dir != "") {
+        sub(/^/, package_dir "/")
+    }
+    print;
+}
diff --git a/lib/tar.bzl b/lib/tar.bzl
@@ -13,25 +13,24 @@ this:
 
 We also provide full control for tar'ring binaries including their runfiles.
 
-## Modifying metadata
+## Mutating the tar contents
 
 The `mtree_spec` rule can be used to create an mtree manifest for the tar file.
-Then you can mutate that spec, as it's just a simple text file, and feed the result
+Then you can mutate that spec using `mtree_mutate` and feed the result
 as the `mtree` attribute of the `tar` rule.
 
-For example, to set the `uid` property, you could:
+For example, to set the owner uid of files in the tar, you could:
 
 ```starlark
 mtree_spec(
     name = "mtree",
     srcs = ["//some:files"],
 )
 
-genrule(
+mtree_mutate(
     name = "change_owner",
-    srcs = ["mtree"],
-    outs = ["mtree.mutated"],
-    cmd = "sed 's/uid=0/uid=1000/' <$< >$@",
+    mtree = ":mtree",
+    owner = "1000",
 )
 
 tar(
@@ -41,10 +40,6 @@ tar(
 )
 ```
 
-Note: We intend to contribute mutation features to https://github.com/vbatts/go-mtree
-to provide a richer API for things like `strip_prefix`.
-In the meantime, see the `lib/tests/tar/BUILD.bazel` file in this repo for examples.
-
 TODO:
 - Provide convenience for rules_pkg users to re-use or replace pkg_files trees
 """
@@ -130,3 +125,43 @@ def tar(name, mtree = "auto", stamp = 0, **kwargs):
         mtree = mtree_target,
         **kwargs
     )
+
+def mtree_mutate(
+        name,
+        mtree,
+        strip_prefix = None,
+        mtime = None,
+        owner = None,
+        ownername = None,
+        awk_script = "@aspect_bazel_lib//lib/private:modify_mtree.awk",
+        **kwargs):
+    """Modify metadata in an mtree file.
+
+    Args:
+        name: name of the target, output will be `[name].mtree`.
+        mtree: input mtree file, typically created by `mtree_spec`.
+        strip_prefix: prefix to remove from all paths in the tar. Files and directories not under this prefix are dropped.
+        mtime: new modification time for all entries.
+        owner: new uid for all entries.
+        ownername: new uname for all entries.
+        awk_script: may be overridden to change the script containing the modification logic.
+        **kwargs: additional named parameters to genrule
+    """
+    vars = []
+    if strip_prefix:
+        vars.append("-v strip_prefix='{}'".format(strip_prefix))
+    if mtime:
+        vars.append("-v mtime='{}'".format(mtime))
+    if owner:
+        vars.append("-v owner='{}'".format(owner))
+    if ownername:
+        vars.append("-v ownername='{}'".format(ownername))
+
+    native.genrule(
+        name = name,
+        srcs = [mtree],
+        outs = [name + ".mtree"],
+        cmd = "awk {} -f $(execpath {}) <$< >$@".format(" ".join(vars), awk_script),
+        tools = [awk_script],
+        **kwargs
+    )
diff --git a/lib/tests/tar/BUILD.bazel b/lib/tests/tar/BUILD.bazel
@@ -1,6 +1,6 @@
 load("@aspect_bazel_lib//lib:copy_directory.bzl", "copy_directory")
 load("@aspect_bazel_lib//lib:diff_test.bzl", "diff_test")
-load("@aspect_bazel_lib//lib:tar.bzl", "mtree_spec", "tar")
+load("@aspect_bazel_lib//lib:tar.bzl", "mtree_mutate", "mtree_spec", "tar")
 load("@aspect_bazel_lib//lib:testing.bzl", "assert_archive_contains")
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
 load("@bazel_skylib//rules:write_file.bzl", "write_file")
@@ -123,25 +123,10 @@ mtree_spec(
     srcs = _SRCS5,
 )
 
-# This is a low-tech way to mutate the mtree specification, just using regex.
-# See docs on tar about future directions for mtree mutation
-genrule(
+mtree_mutate(
     name = "strip_prefix",
-    srcs = ["mtree5"],
-    outs = ["mtree5.stripped"],
-    # Modify lines starting with the package name, e.g.
-    # lib/tests/tar/a uid=0 gid=0 time=1672560000 mode=0755 type=file content=bazel-out/darwin_arm64-opt/bin/lib/tests/tar/a
-    # ->
-    # a uid=0 gid=0 time=1672560000 mode=0755 type=file content=bazel-out/darwin_arm64-opt/bin/lib/tests/tar/a
-    cmd = "sed '{}' <$< | sed '/^\\ /d' > $@".format(
-        "; ".join(reversed([
-            "s#^{s}/##; s#^{s}##".format(s = "/".join(package_name().split("/")[:i]))
-            for (i, _) in enumerate(
-                package_name().split("/"),
-                1,
-            )
-        ])),
-    ),
+    mtree = "mtree5",
+    strip_prefix = package_name(),
 )
 
 tar(
@@ -342,3 +327,30 @@ assert_tar_listing(
         "drwxrwxrwt  0 0      0           0 Aug  3  2017 ./tmp/",
     ],
 )
+
+# Case 12: arbitrary mtree modifications
+mtree_mutate(
+    name = "modified1",
+    mtree = "source-casync.mtree",
+    strip_prefix = "xattr",
+)
+
+diff_test(
+    name = "test1",
+    file1 = "modified1.mtree",
+    file2 = "expected1.mtree",
+)
+
+mtree_mutate(
+    name = "modified2",
+    mtime = 946684740,  # 1999-12-31, 23:59
+    mtree = "source-casync.mtree",
+    owner = "123",
+    ownername = "fred",
+)
+
+diff_test(
+    name = "test2",
+    file1 = "modified2.mtree",
+    file2 = "expected2.mtree",
+)
diff --git a/lib/tests/tar/expected1.mtree b/lib/tests/tar/expected1.mtree
@@ -0,0 +1,4 @@
+xattr.go type=file mode=0664 size=984 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=a2700b603df30c3b0a91bdcf9045e64aea42f62647b0128ea154f51a0c48991e
+xattr_test.go type=file mode=0664 size=859 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=91294ea554801b75f6a9e33c268807c9620b531eb813ea24512dd4eeaf0592e4
+xattr_unsupported.go type=file mode=0664 size=509 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=81ced06a1cdf88c4936d10bbf8d46edc2d42dc26efeed3691ddbeeb469865f8a
+xattr_unsupported_test.go type=file mode=0664 size=877 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=46605a03a985c7a3a4ab1a488f81382db4865f77b90b6a2b32693af39a8e1fba
diff --git a/lib/tests/tar/expected2.mtree b/lib/tests/tar/expected2.mtree
@@ -0,0 +1,5 @@
+xattr type=dir mode=0775 uid=123 gid=1000 uname=fred gname=vbatts time=946684740
+xattr/xattr.go type=file mode=0664 size=984 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=a2700b603df30c3b0a91bdcf9045e64aea42f62647b0128ea154f51a0c48991e
+xattr/xattr_test.go type=file mode=0664 size=859 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=91294ea554801b75f6a9e33c268807c9620b531eb813ea24512dd4eeaf0592e4
+xattr/xattr_unsupported.go type=file mode=0664 size=509 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=81ced06a1cdf88c4936d10bbf8d46edc2d42dc26efeed3691ddbeeb469865f8a
+xattr/xattr_unsupported_test.go type=file mode=0664 size=877 uid=123 gid=1000 uname=fred gname=vbatts time=946684740 sha512256digest=46605a03a985c7a3a4ab1a488f81382db4865f77b90b6a2b32693af39a8e1fba
diff --git a/lib/tests/tar/source-casync.mtree b/lib/tests/tar/source-casync.mtree
@@ -0,0 +1,5 @@
+xattr type=dir mode=0775 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767
+xattr/xattr.go type=file mode=0664 size=984 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=a2700b603df30c3b0a91bdcf9045e64aea42f62647b0128ea154f51a0c48991e
+xattr/xattr_test.go type=file mode=0664 size=859 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=91294ea554801b75f6a9e33c268807c9620b531eb813ea24512dd4eeaf0592e4
+xattr/xattr_unsupported.go type=file mode=0664 size=509 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=81ced06a1cdf88c4936d10bbf8d46edc2d42dc26efeed3691ddbeeb469865f8a
+xattr/xattr_unsupported_test.go type=file mode=0664 size=877 uid=1000 gid=1000 uname=vbatts gname=vbatts time=1512774394.470900767 sha512256digest=46605a03a985c7a3a4ab1a488f81382db4865f77b90b6a2b32693af39a8e1fba