Skip to content

Commit

Permalink
Write a repo mapping manifest in the runfiles directory
Browse files Browse the repository at this point in the history
To ensure we can use repo mappings in the runfiles library, this change writes an extra file "my_binary_target.runfiles/_repo_mapping", which contains a bunch of (base_repo_canonical_name, apparent_repo_name, canonical_repo_name) triples. See https://github.com/bazelbuild/proposals/blob/main/designs/2022-07-21-locating-runfiles-with-bzlmod.md for more information.

Work towards #16124

PiperOrigin-RevId: 475820334
Change-Id: I885b4df093bd2c783c57d19f995f420b9b29b53c
  • Loading branch information
Wyverald authored and Copybara-Service committed Sep 21, 2022
1 parent 2ed1f3b commit 0bcf791
Show file tree
Hide file tree
Showing 6 changed files with 524 additions and 6 deletions.
20 changes: 20 additions & 0 deletions src/main/java/com/google/devtools/build/lib/analysis/BUILD
Expand Up @@ -336,6 +336,7 @@ java_library(
":package_specification_provider",
":platform_options",
":provider_collection",
":repo_mapping_manifest_action",
":required_config_fragments_provider",
":resolved_toolchain_context",
":rule_configured_object_value",
Expand Down Expand Up @@ -982,6 +983,25 @@ java_library(
],
)

java_library(
name = "repo_mapping_manifest_action",
srcs = ["RepoMappingManifestAction.java"],
deps = [
":actions/abstract_file_write_action",
":actions/deterministic_writer",
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
"//src/main/java/com/google/devtools/build/lib/actions:commandline_item",
"//src/main/java/com/google/devtools/build/lib/cmdline",
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
"//src/main/java/com/google/devtools/build/lib/util",
"//src/main/java/com/google/devtools/build/lib/vfs:ospathpolicy",
"//src/main/java/net/starlark/java/eval",
"//third_party:guava",
"//third_party:jsr305",
],
)

java_library(
name = "required_config_fragments_provider",
srcs = ["RequiredConfigFragmentsProvider.java"],
Expand Down
@@ -0,0 +1,122 @@
// Copyright 2022 The Bazel Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package com.google.devtools.build.lib.analysis;

import static java.nio.charset.StandardCharsets.ISO_8859_1;

import com.google.common.collect.ImmutableSortedMap;
import com.google.common.collect.Maps;
import com.google.devtools.build.lib.actions.ActionExecutionContext;
import com.google.devtools.build.lib.actions.ActionKeyContext;
import com.google.devtools.build.lib.actions.ActionOwner;
import com.google.devtools.build.lib.actions.Artifact;
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
import com.google.devtools.build.lib.actions.ExecException;
import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
import com.google.devtools.build.lib.analysis.actions.DeterministicWriter;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.collect.nestedset.Order;
import com.google.devtools.build.lib.util.Fingerprint;
import com.google.devtools.build.lib.vfs.OsPathPolicy;
import java.io.BufferedWriter;
import java.io.OutputStreamWriter;
import java.io.Writer;
import java.util.Map;
import java.util.Map.Entry;
import java.util.UUID;
import javax.annotation.Nullable;
import net.starlark.java.eval.EvalException;

/** Creates a manifest file describing the repos and mappings relevant for a runfile tree. */
public class RepoMappingManifestAction extends AbstractFileWriteAction {
private static final UUID MY_UUID = UUID.fromString("458e351c-4d30-433d-b927-da6cddd4737f");

private final ImmutableSortedMap<RepositoryName, ImmutableSortedMap<String, RepositoryName>>
reposAndMappings;
private final String workspaceName;

public RepoMappingManifestAction(
ActionOwner owner,
Artifact output,
Map<RepositoryName, RepositoryMapping> reposAndMappings,
String workspaceName) {
super(owner, NestedSetBuilder.emptySet(Order.STABLE_ORDER), output, /*makeExecutable=*/ false);
this.reposAndMappings =
ImmutableSortedMap.copyOf(
Maps.transformValues(
reposAndMappings, repoMapping -> ImmutableSortedMap.copyOf(repoMapping.entries())),
(a, b) -> OsPathPolicy.getFilePathOs().compare(a.getName(), b.getName()));
this.workspaceName = workspaceName;
}

@Override
public String getMnemonic() {
return "RepoMappingManifest";
}

@Override
protected String getRawProgressMessage() {
return "writing repo mapping manifest for " + getOwner().getLabel();
}

@Override
protected void computeKey(
ActionKeyContext actionKeyContext,
@Nullable ArtifactExpander artifactExpander,
Fingerprint fp)
throws CommandLineExpansionException, EvalException, InterruptedException {
fp.addUUID(MY_UUID);
fp.addString(workspaceName);
for (Entry<RepositoryName, ImmutableSortedMap<String, RepositoryName>> repoAndMapping :
reposAndMappings.entrySet()) {
fp.addString(repoAndMapping.getKey().getName());
fp.addInt(repoAndMapping.getValue().size());
for (Entry<String, RepositoryName> mappingEntry : repoAndMapping.getValue().entrySet()) {
fp.addString(mappingEntry.getKey());
fp.addString(mappingEntry.getValue().getName());
}
}
}

@Override
public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx)
throws InterruptedException, ExecException {
return out -> {
Writer writer = new BufferedWriter(new OutputStreamWriter(out, ISO_8859_1));
for (Entry<RepositoryName, ImmutableSortedMap<String, RepositoryName>> repoAndMapping :
reposAndMappings.entrySet()) {
for (Entry<String, RepositoryName> mappingEntry : repoAndMapping.getValue().entrySet()) {
if (mappingEntry.getKey().isEmpty()) {
// The apparent repo name can only be empty for the main repo. We skip this line.
continue;
}
writer.write(repoAndMapping.getKey().getName());
writer.write(',');
writer.write(mappingEntry.getKey());
writer.write(',');
if (mappingEntry.getValue().isMain()) {
writer.write(workspaceName);
} else {
writer.write(mappingEntry.getValue().getName());
}
writer.write('\n');
}
}
writer.flush();
};
}
}
Expand Up @@ -25,15 +25,20 @@
import com.google.devtools.build.lib.analysis.actions.SymlinkTreeAction;
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
import com.google.devtools.build.lib.analysis.config.RunUnder;
import com.google.devtools.build.lib.cmdline.RepositoryMapping;
import com.google.devtools.build.lib.cmdline.RepositoryName;
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
import com.google.devtools.build.lib.packages.Package;
import com.google.devtools.build.lib.packages.TargetUtils;
import com.google.devtools.build.lib.packages.Type;
import com.google.devtools.build.lib.vfs.FileSystemUtils;
import com.google.devtools.build.lib.vfs.Path;
import com.google.devtools.build.lib.vfs.PathFragment;
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Map;
Expand Down Expand Up @@ -76,11 +81,13 @@ public final class RunfilesSupport {
private static final String RUNFILES_DIR_EXT = ".runfiles";
private static final String INPUT_MANIFEST_EXT = ".runfiles_manifest";
private static final String OUTPUT_MANIFEST_BASENAME = "MANIFEST";
private static final String REPO_MAPPING_MANIFEST_BASENAME = "_repo_mapping";

private final Runfiles runfiles;

private final Artifact runfilesInputManifest;
private final Artifact runfilesManifest;
private final Artifact repoMappingManifest;
private final Artifact runfilesMiddleman;
private final Artifact owningExecutable;
private final boolean buildRunfileLinks;
Expand Down Expand Up @@ -132,15 +139,18 @@ private static RunfilesSupport create(
runfilesInputManifest = null;
runfilesManifest = null;
}
Artifact repoMappingManifest = createRepoMappingManifestAction(ruleContext, owningExecutable);
Artifact runfilesMiddleman =
createRunfilesMiddleman(ruleContext, owningExecutable, runfiles, runfilesManifest);
createRunfilesMiddleman(
ruleContext, owningExecutable, runfiles, runfilesManifest, repoMappingManifest);

boolean runfilesEnabled = ruleContext.getConfiguration().runfilesEnabled();

return new RunfilesSupport(
runfiles,
runfilesInputManifest,
runfilesManifest,
repoMappingManifest,
runfilesMiddleman,
owningExecutable,
buildRunfileLinks,
Expand All @@ -153,6 +163,7 @@ private RunfilesSupport(
Runfiles runfiles,
Artifact runfilesInputManifest,
Artifact runfilesManifest,
Artifact repoMappingManifest,
Artifact runfilesMiddleman,
Artifact owningExecutable,
boolean buildRunfileLinks,
Expand All @@ -162,6 +173,7 @@ private RunfilesSupport(
this.runfiles = runfiles;
this.runfilesInputManifest = runfilesInputManifest;
this.runfilesManifest = runfilesManifest;
this.repoMappingManifest = repoMappingManifest;
this.runfilesMiddleman = runfilesMiddleman;
this.owningExecutable = owningExecutable;
this.buildRunfileLinks = buildRunfileLinks;
Expand Down Expand Up @@ -268,6 +280,16 @@ public Artifact getRunfilesManifest() {
return runfilesManifest;
}

/**
* Returns the foo.runfiles/_repo_mapping file if Bazel is run with transitive package tracking
* turned on (see {@code SkyframeExecutor#getForcedSingleSourceRootIfNoExecrootSymlinkCreation}).
* Otherwise, returns null.
*/
@Nullable
public Artifact getRepoMappingManifest() {
return repoMappingManifest;
}

/** Returns the root directory of the runfiles symlink farm; otherwise, returns null. */
@Nullable
public Path getRunfilesDirectory() {
Expand Down Expand Up @@ -327,12 +349,16 @@ private static Artifact createRunfilesMiddleman(
ActionConstructionContext context,
Artifact owningExecutable,
Runfiles runfiles,
@Nullable Artifact runfilesManifest) {
@Nullable Artifact runfilesManifest,
Artifact repoMappingManifest) {
NestedSetBuilder<Artifact> deps = NestedSetBuilder.stableOrder();
deps.addTransitive(runfiles.getAllArtifacts());
if (runfilesManifest != null) {
deps.add(runfilesManifest);
}
if (repoMappingManifest != null) {
deps.add(repoMappingManifest);
}
return context
.getAnalysisEnvironment()
.getMiddlemanFactory()
Expand Down Expand Up @@ -495,4 +521,56 @@ public static Path inputManifestPath(Path runfilesDir) {
public static Path outputManifestPath(Path runfilesDir) {
return runfilesDir.getRelative(OUTPUT_MANIFEST_BASENAME);
}

@Nullable
private static Artifact createRepoMappingManifestAction(
RuleContext ruleContext, Artifact owningExecutable) {
NestedSet<Package> transitivePackages =
ruleContext.getTransitivePackagesForRunfileRepoMappingManifest();
if (transitivePackages == null) {
// For environments where transitive packages are not tracked, we don't have external repos,
// so don't build the repo mapping manifest in such cases.
return null;
}
Set<NestedSet.Node> visitedNestedSets = new HashSet<>();
Map<RepositoryName, RepositoryMapping> reposAndMappings = new HashMap<>();
collectReposAndMappings(transitivePackages, visitedNestedSets, reposAndMappings);

PathFragment executablePath =
(owningExecutable != null)
? owningExecutable.getOutputDirRelativePath(
ruleContext.getConfiguration().isSiblingRepositoryLayout())
: ruleContext.getPackageDirectory().getRelative(ruleContext.getLabel().getName());
Artifact repoMappingManifest =
ruleContext.getDerivedArtifact(
executablePath
.replaceName(executablePath.getBaseName() + RUNFILES_DIR_EXT)
.getRelative(REPO_MAPPING_MANIFEST_BASENAME),
ruleContext.getBinDirectory());
ruleContext
.getAnalysisEnvironment()
.registerAction(
new RepoMappingManifestAction(
ruleContext.getActionOwner(),
repoMappingManifest,
reposAndMappings,
ruleContext.getWorkspaceName()));
return repoMappingManifest;
}

private static void collectReposAndMappings(
NestedSet<Package> transitivePackages,
Set<NestedSet.Node> visitedNestedSets,
Map<RepositoryName, RepositoryMapping> reposAndMappings) {
if (!visitedNestedSets.add(transitivePackages.toNode())) {
return;
}
for (Package pkg : transitivePackages.getLeaves()) {
reposAndMappings.putIfAbsent(
pkg.getPackageIdentifier().getRepository(), pkg.getRepositoryMapping());
}
for (NestedSet<Package> transitives : transitivePackages.getNonLeaves()) {
collectReposAndMappings(transitives, visitedNestedSets, reposAndMappings);
}
}
}
Expand Up @@ -35,7 +35,8 @@ public abstract class RepositoryMapping {
// Always fallback to the requested name
public static final RepositoryMapping ALWAYS_FALLBACK = createAllowingFallback(ImmutableMap.of());

abstract ImmutableMap<String, RepositoryName> repositoryMapping();
/** Returns all the entries in this repo mapping. */
public abstract ImmutableMap<String, RepositoryName> entries();

/**
* The owner repo of this repository mapping. It is for providing useful debug information when
Expand Down Expand Up @@ -64,7 +65,7 @@ public static RepositoryMapping createAllowingFallback(
*/
public RepositoryMapping withAdditionalMappings(Map<String, RepositoryName> additionalMappings) {
HashMap<String, RepositoryName> allMappings = new HashMap<>(additionalMappings);
allMappings.putAll(repositoryMapping());
allMappings.putAll(entries());
return new AutoValue_RepositoryMapping(ImmutableMap.copyOf(allMappings), ownerRepo());
}

Expand All @@ -74,15 +75,15 @@ public RepositoryMapping withAdditionalMappings(Map<String, RepositoryName> addi
* repo of the given additional mappings is ignored.
*/
public RepositoryMapping withAdditionalMappings(RepositoryMapping additionalMappings) {
return withAdditionalMappings(additionalMappings.repositoryMapping());
return withAdditionalMappings(additionalMappings.entries());
}

/**
* Returns the canonical repository name associated with the given apparent repo name. The
* provided apparent repo name is assumed to be valid.
*/
public RepositoryName get(String preMappingName) {
RepositoryName canonicalRepoName = repositoryMapping().get(preMappingName);
RepositoryName canonicalRepoName = entries().get(preMappingName);
if (canonicalRepoName != null) {
return canonicalRepoName;
}
Expand Down
23 changes: 23 additions & 0 deletions src/test/java/com/google/devtools/build/lib/analysis/BUILD
Expand Up @@ -35,6 +35,7 @@ java_library(
"JDKJavaLauncherRunfilesSupportTest.java",
"PackageGroupBuildViewTest.java",
"RuleConfiguredTargetTest.java",
"RunfilesRepoMappingManifestTest.java",
"SourceManifestActionTest.java",
"AnalysisFailureInfoTest.java",
],
Expand Down Expand Up @@ -355,6 +356,28 @@ java_test(
],
)

java_test(
name = "RunfilesRepoMappingManifestTest",
srcs = ["RunfilesRepoMappingManifestTest.java"],
deps = [
"//src/main/java/com/google/devtools/build/lib/actions",
"//src/main/java/com/google/devtools/build/lib/analysis:blaze_directories",
"//src/main/java/com/google/devtools/build/lib/analysis:repo_mapping_manifest_action",
"//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:resolution_impl",
"//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
"//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
"//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
"//src/main/java/com/google/devtools/build/lib/vfs",
"//src/main/java/com/google/devtools/build/skyframe",
"//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
"//src/test/java/com/google/devtools/build/lib/analysis/util",
"//src/test/java/com/google/devtools/build/lib/bazel/bzlmod:util",
"//third_party:guava",
"//third_party:junit4",
"//third_party:truth",
],
)

java_test(
name = "TransitiveValidationPropagationTest",
srcs = ["TransitiveValidationPropagationTest.java"],
Expand Down

0 comments on commit 0bcf791

Please sign in to comment.