Migrate to an incremental format for MODULE.bazel.lock

The old lockfile fields and all code related to it as well as the workarounds for local path inclusions are removed.

The distribution archive lockfile needs to be checked in temporarily as the main `MODULE.bazel.lock` file does not contain the hashes for the registry files.

Fixes #19621
Fixes #20369

RELNOTES: The format for MODULE.bazel.lock is now less likely to result in merge conflicts and is updated incrementally, with only new files downloaded from registries and existing ones taken from the repository cache (if configured).

Closes #22154.

PiperOrigin-RevId: 633233519
Change-Id: Ie2c3042e4141a36e472b2c25cbd67be4aad096a1

.bazelci/presubmit.yml

@@ -428,6 +428,7 @@ tasks:
       - "-//src/test/py/bazel:bazel_yanked_versions_test"
       - "-//src/test/py/bazel:bzlmod_query_test"
       - "-//src/test/py/bazel:mod_command_test"
+      - "-//src/test/shell/bazel:starlark_repository_test"
       - "-//src/test/shell/bazel:verify_workspace"
       # Flaky on rbe_ubuntu2004
       # https://github.com/bazelbuild/continuous-integration/issues/1631

MODULE.bazel

@@ -33,6 +33,9 @@ bazel_dep(name = "rules_testing", version = "0.6.0")
 bazel_dep(name = "googletest", version = "1.14.0", repo_name = "com_google_googletest")
 bazel_dep(name = "with_cfg.bzl", version = "0.2.4")
 
+# TODO(fmeum): Remove the dependency on buildozer after Bazel is built with 7.2.0.
+bazel_dep(name = "buildozer", version = "7.1.1.1")
+
 # TODO(pcloudy): Add remoteapis and googleapis as Bazel modules in the BCR.
 bazel_dep(name = "remoteapis", version = "")
 bazel_dep(name = "googleapis", version = "")

distdir.bzl

@@ -13,7 +13,7 @@
 # limitations under the License.
 """Defines a repository rule that generates an archive consisting of the specified files to fetch"""
 
-load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts")
+load("//src/tools/bzlmod:utils.bzl", "parse_http_artifacts", "parse_registry_files")
 
 _BUILD = """
 load("@rules_pkg//pkg:tar.bzl", "pkg_tar")
@@ -87,10 +87,11 @@ def _repo_cache_tar_impl(ctx):
     """
     lockfile_path = ctx.path(ctx.attr.lockfile)
     http_artifacts = parse_http_artifacts(ctx, lockfile_path, ctx.attr.repos)
+    registry_files = parse_registry_files(ctx, lockfile_path, ctx.attr.module_files)
 
     archive_files = []
     readme_content = "This directory contains repository cache artifacts for the following URLs:\n\n"
-    for artifact in http_artifacts:
+    for artifact in http_artifacts + registry_files:
         url = artifact["url"]
         if "integrity" in artifact:
             # ./tempfile could be a hard link if --experimental_repository_cache_hardlinks is used,
@@ -122,6 +123,7 @@ _repo_cache_tar_attrs = {
     "lockfile": attr.label(default = Label("//:MODULE.bazel.lock")),
     "dirname": attr.string(default = "repository_cache"),
     "repos": attr.string_list(),
+    "module_files": attr.label_list(),
 }
 
 repo_cache_tar = repository_rule(

extensions.bzl

@@ -29,7 +29,17 @@ def _bazel_build_deps(_ctx):
     _ctx.path(Label("//:MODULE.bazel"))  # Make sure the `bootstrap_repo_cache` repo is updated when MODULE.bazel changes.
     embedded_jdk_repositories()
     debian_deps()
-    repo_cache_tar(name = "bootstrap_repo_cache", repos = DIST_ARCHIVE_REPOS, dirname = "derived/repository_cache")
+    repo_cache_tar(
+        name = "bootstrap_repo_cache",
+        repos = DIST_ARCHIVE_REPOS,
+        dirname = "derived/repository_cache",
+        module_files = [
+            "//:MODULE.bazel",
+            "//third_party/googleapis:MODULE.bazel",
+            "//third_party/remoteapis:MODULE.bazel",
+            "//src:MODULE.tools",
+        ],
+    )
     BAZEL_TOOLS_DEPS_REPOS = parse_bazel_module_repos(_ctx, _ctx.path(Label("//src/test/tools/bzlmod:MODULE.bazel.lock")))
     repo_cache_tar(name = "bazel_tools_repo_cache", repos = BAZEL_TOOLS_DEPS_REPOS, lockfile = "//src/test/tools/bzlmod:MODULE.bazel.lock")
     distdir_tar(name = "workspace_repo_cache", dist_deps = WORKSPACE_REPOS)

repositories.bzl

@@ -24,21 +24,23 @@ load("//src/tools/bzlmod:utils.bzl", "get_canonical_repo_name")
 #
 ##################################################################################
 DIST_ARCHIVE_REPOS = [get_canonical_repo_name(repo) for repo in [
+    # keep sorted
     "abseil-cpp",
     "apple_support",
     "bazel_skylib",
     "blake3",
+    "buildozer",
     "c-ares",
     "com_github_grpc_grpc",
     "com_google_protobuf",
     "io_bazel_skydoc",
     "platforms",
     "rules_cc",
     "rules_go",
+    "rules_graalvm",
     "rules_java",
     "rules_jvm_external",
     "rules_kotlin",
-    "rules_graalvm",
     "rules_license",
     "rules_pkg",
     "rules_proto",

src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java

@@ -270,8 +270,8 @@ SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace
         .addSkyFunction(
             SkyFunctions.REGISTRY,
             new RegistryFunction(
-                new RegistryFactoryImpl(
-                    directories.getWorkspace(), downloadManager, clientEnvironmentSupplier)))
+                new RegistryFactoryImpl(downloadManager, clientEnvironmentSupplier),
+                directories.getWorkspace()))
         .addSkyFunction(SkyFunctions.REPO_SPEC, new RepoSpecFunction())
         .addSkyFunction(SkyFunctions.YANKED_VERSIONS, new YankedVersionsFunction())
         .addSkyFunction(

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -79,12 +79,13 @@ java_library(
     ],
     deps = [
         ":common",
+        ":yanked_versions_value",
+        "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/cache",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/events",
         "//src/main/java/com/google/devtools/build/lib/profiler",
         "//src/main/java/com/google/devtools/build/lib/util:os",
-        "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//third_party:gson",
@@ -101,10 +102,11 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib:runtime",
         "//src/main/java/com/google/devtools/build/lib/bazel/bzlmod:module_extension",
         "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
-        "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:skyframe_cluster",
         "//src/main/java/com/google/devtools/build/lib/vfs",
+        "//src/main/java/com/google/devtools/build/skyframe",
         "//third_party:flogger",
         "//third_party:guava",
         "//third_party:jsr305",
@@ -119,9 +121,7 @@ java_library(
         "BazelDepGraphValue.java",
         "BazelFetchAllValue.java",
         "BazelLockFileValue.java",
-        "BazelModuleResolutionEvent.java",
         "BazelModuleResolutionValue.java",
-        "BzlmodFlagsAndEnvVars.java",
         "CompiledModuleFile.java",
         "GitOverride.java",
         "InterimModule.java",
@@ -188,7 +188,6 @@ java_library(
         "BazelFetchAllFunction.java",
         "BazelLockFileFunction.java",
         "BazelModuleResolutionFunction.java",
-        "BzlmodFlagsAndEnvVars.java",
         "DelegateTypeAdapterFactory.java",
         "Discovery.java",
         "GsonTypeAdapterUtil.java",
@@ -298,12 +297,10 @@ java_library(
     deps = [
         ":resolution",
         ":root_module_file_fixup",
-        "//src/main/java/com/google/devtools/build/lib/bazel/repository:repository_options",
         "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
         "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec:serialization-constant",
         "//src/main/java/com/google/devtools/build/lib/vfs",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
-        "//src/main/java/net/starlark/java/eval",
         "//third_party:auto_value",
         "//third_party:guava",
     ],
@@ -316,20 +313,15 @@ java_library(
     ],
     deps = [
         ":common",
-        ":exception",
         ":resolution",
-        ":resolution_impl",
         ":root_module_file_fixup",
         ":tidy",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/rules:repository/repository_function",
-        "//src/main/java/com/google/devtools/build/lib/skyframe:precomputed_value",
         "//src/main/java/com/google/devtools/build/lib/skyframe:repository_mapping_value",
         "//src/main/java/com/google/devtools/build/lib/vfs",
-        "//src/main/java/com/google/devtools/build/skyframe",
         "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
         "//src/main/java/net/starlark/java/eval",
-        "//src/main/protobuf:failure_details_java_proto",
         "//third_party:guava",
         "//third_party:jsr305",
     ],
@@ -401,3 +393,16 @@ java_library(
         "//third_party:guava",
     ],
 )
+
+java_library(
+    name = "yanked_versions_value",
+    srcs = ["YankedVersionsValue.java"],
+    deps = [
+        ":common",
+        "//src/main/java/com/google/devtools/build/lib/skyframe:sky_functions",
+        "//src/main/java/com/google/devtools/build/lib/skyframe/serialization/autocodec",
+        "//src/main/java/com/google/devtools/build/skyframe:skyframe-objects",
+        "//third_party:auto_value",
+        "//third_party:guava",
+    ],
+)

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BazelDepGraphFunction.java

@@ -15,40 +15,31 @@
 
 package com.google.devtools.build.lib.bazel.bzlmod;
 
-import static com.google.common.base.Strings.nullToEmpty;
 import static com.google.common.collect.ImmutableBiMap.toImmutableBiMap;
 import static com.google.common.collect.ImmutableList.toImmutableList;
-import static com.google.common.collect.ImmutableMap.toImmutableMap;
 import static com.google.common.collect.ImmutableSet.toImmutableSet;
 import static java.util.stream.Collectors.counting;
 import static java.util.stream.Collectors.groupingBy;
 
-import com.google.common.annotations.VisibleForTesting;
 import com.google.common.base.Preconditions;
 import com.google.common.collect.BiMap;
 import com.google.common.collect.HashBiMap;
 import com.google.common.collect.ImmutableBiMap;
-import com.google.common.collect.ImmutableList;
 import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.ImmutableTable;
-import com.google.devtools.build.lib.bazel.bzlmod.ModuleFileValue.RootModuleFileValue;
-import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
 import com.google.devtools.build.lib.cmdline.LabelSyntaxException;
 import com.google.devtools.build.lib.cmdline.PackageIdentifier;
 import com.google.devtools.build.lib.cmdline.RepositoryMapping;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.packages.LabelConverter;
 import com.google.devtools.build.lib.server.FailureDetails.ExternalDeps.Code;
-import com.google.devtools.build.lib.skyframe.ClientEnvironmentFunction;
-import com.google.devtools.build.lib.skyframe.ClientEnvironmentValue;
 import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.devtools.build.skyframe.SkyFunction;
 import com.google.devtools.build.skyframe.SkyFunctionException;
 import com.google.devtools.build.skyframe.SkyFunctionException.Transience;
 import com.google.devtools.build.skyframe.SkyKey;
 import com.google.devtools.build.skyframe.SkyValue;
-import java.util.Map;
 import java.util.Map.Entry;
 import javax.annotation.Nullable;
 
@@ -65,59 +56,12 @@ public BazelDepGraphFunction() {}
   @Nullable
   public SkyValue compute(SkyKey skyKey, Environment env)
       throws BazelDepGraphFunctionException, InterruptedException {
-    RootModuleFileValue root =
-        (RootModuleFileValue) env.getValue(ModuleFileValue.KEY_FOR_ROOT_MODULE);
-    if (root == null) {
+    BazelModuleResolutionValue selectionResult =
+        (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY);
+    if (env.valuesMissing()) {
       return null;
     }
-    LockfileMode lockfileMode = BazelLockFileFunction.LOCKFILE_MODE.get(env);
-
-    ImmutableMap<String, String> localOverrideHashes = null;
-    ImmutableMap<ModuleKey, Module> depGraph = null;
-    BzlmodFlagsAndEnvVars flags = null;
-    BazelLockFileValue lockfile = null;
-
-    // If the module has not changed (has the same contents and flags as the lockfile),
-    // read the dependency graph from the lock file, else run resolution and update lockfile
-    if (!lockfileMode.equals(LockfileMode.OFF)) {
-      lockfile = (BazelLockFileValue) env.getValue(BazelLockFileValue.KEY);
-      if (lockfile == null) {
-        return null;
-      }
-      flags = getFlagsAndEnvVars(env);
-      if (flags == null) { // unable to read environment variables
-        return null;
-      }
-      localOverrideHashes = getLocalOverridesHashes(root.getOverrides(), env);
-      if (localOverrideHashes == null) { // still reading an override "module"
-        return null;
-      }
-
-      if (root.getModuleFileHash().equals(lockfile.getModuleFileHash())
-          && flags.equals(lockfile.getFlags())
-          && localOverrideHashes.equals(lockfile.getLocalOverrideHashes())) {
-        depGraph = lockfile.getModuleDepGraph();
-      } else if (lockfileMode.equals(LockfileMode.ERROR)) {
-        ImmutableList<String> diffLockfile =
-            lockfile.getModuleAndFlagsDiff(root.getModuleFileHash(), localOverrideHashes, flags);
-        throw new BazelDepGraphFunctionException(
-            ExternalDepsException.withMessage(
-                Code.BAD_MODULE,
-                "MODULE.bazel.lock is no longer up-to-date because: %s. "
-                    + "Please run `bazel mod deps --lockfile_mode=update` to update your lockfile.",
-                String.join(", ", diffLockfile)),
-            Transience.PERSISTENT);
-      }
-    }
-
-    if (depGraph == null) {
-      BazelModuleResolutionValue selectionResult =
-          (BazelModuleResolutionValue) env.getValue(BazelModuleResolutionValue.KEY);
-      if (env.valuesMissing()) {
-        return null;
-      }
-      depGraph = selectionResult.getResolvedDepGraph();
-    }
+    var depGraph = selectionResult.getResolvedDepGraph();
 
     ImmutableBiMap<RepositoryName, ModuleKey> canonicalRepoNameLookup =
         computeCanonicalRepoNameLookup(depGraph);
@@ -131,20 +75,6 @@ public SkyValue compute(SkyKey skyKey, Environment env)
     ImmutableBiMap<String, ModuleExtensionId> extensionUniqueNames =
         calculateUniqueNameForUsedExtensionId(extensionUsagesById);
 
-    if (lockfileMode.equals(LockfileMode.UPDATE)) {
-      BazelLockFileValue resolutionOnlyLockfile =
-          BazelLockFileValue.builder()
-              .setModuleFileHash(root.getModuleFileHash())
-              .setFlags(flags)
-              .setLocalOverrideHashes(localOverrideHashes)
-              .setModuleDepGraph(depGraph)
-              .build();
-      env.getListener()
-          .post(
-              BazelModuleResolutionEvent.create(
-                  lockfile, resolutionOnlyLockfile, extensionUsagesById));
-    }
-
     return BazelDepGraphValue.create(
         depGraph,
         canonicalRepoNameLookup,
@@ -153,63 +83,6 @@ public SkyValue compute(SkyKey skyKey, Environment env)
         extensionUniqueNames.inverse());
   }
 
-  @Nullable
-  @VisibleForTesting
-  static ImmutableMap<String, String> getLocalOverridesHashes(
-      Map<String, ModuleOverride> overrides, Environment env) throws InterruptedException {
-    ImmutableMap.Builder<String, String> localOverrideHashes = new ImmutableMap.Builder<>();
-    for (Entry<String, ModuleOverride> entry : overrides.entrySet()) {
-      if (entry.getValue() instanceof LocalPathOverride) {
-        ModuleFileValue moduleValue =
-            (ModuleFileValue)
-                env.getValue(
-                    ModuleFileValue.key(
-                        ModuleKey.create(entry.getKey(), Version.EMPTY), entry.getValue()));
-        if (moduleValue == null) {
-          return null;
-        }
-        localOverrideHashes.put(entry.getKey(), moduleValue.getModuleFileHash());
-      }
-    }
-    return localOverrideHashes.buildOrThrow();
-  }
-
-  @VisibleForTesting
-  @Nullable
-  static BzlmodFlagsAndEnvVars getFlagsAndEnvVars(Environment env) throws InterruptedException {
-    ClientEnvironmentValue allowedYankedVersionsFromEnv =
-        (ClientEnvironmentValue)
-            env.getValue(
-                ClientEnvironmentFunction.key(
-                    YankedVersionsUtil.BZLMOD_ALLOWED_YANKED_VERSIONS_ENV));
-    if (allowedYankedVersionsFromEnv == null) {
-      return null;
-    }
-
-    ImmutableMap<String, String> moduleOverrides =
-        ModuleFileFunction.MODULE_OVERRIDES.get(env).entrySet().stream()
-            .collect(
-                toImmutableMap(Entry::getKey, e -> ((LocalPathOverride) e.getValue()).getPath()));
-
-    ImmutableList<String> yankedVersions =
-        ImmutableList.copyOf(YankedVersionsUtil.ALLOWED_YANKED_VERSIONS.get(env));
-    Boolean ignoreDevDeps = ModuleFileFunction.IGNORE_DEV_DEPS.get(env);
-    String compatabilityMode =
-        BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE.get(env).name();
-    String directDepsMode = BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES.get(env).name();
-
-    String envYanked = allowedYankedVersionsFromEnv.getValue();
-
-    return BzlmodFlagsAndEnvVars.create(
-        ModuleFileFunction.REGISTRIES.get(env),
-        moduleOverrides,
-        yankedVersions,
-        nullToEmpty(envYanked),
-        ignoreDevDeps,
-        directDepsMode,
-        compatabilityMode);
-  }
-
   private static ImmutableTable<ModuleExtensionId, ModuleKey, ModuleExtensionUsage>
       getExtensionUsagesById(
           ImmutableMap<ModuleKey, Module> depGraph,