Support local_repository in Bazel Registry's source.json file

Add "module_base_path" to BazelRegistryJson
Add type to SourceJson class that defaults to "archive" and can be changed trough the file to "local_repository".
If type is local_repository, calculate path as following:
- If path is relative, use module_base_path/path.
- If module_base_path is relative, use registry_url/module_base_path/path (registry_url has to start with file://) else would produce an error

fixes #16319

RELNOTES[NEW]: Support local_repository in Bazel Registry's source.json file

PiperOrigin-RevId: 481660476
Change-Id: Ib12944e416c19ac885c1088b888014acc7f5d047

site/en/build/bzlmod.md

@@ -221,9 +221,10 @@ version. Notably, it does *not* need to serve the source archives itself.
 
 An index registry must follow the format below:
 
-*   `/bazel_registry.json`: A JSON file containing metadata for the registry.
-    Currently, it only has one key, `mirrors`, specifying the list of mirrors to
-    use for source archives.
+*   `/bazel_registry.json`: A JSON file containing metadata for the registry like:
+    * `mirrors`, specifying the list of mirrors to use for source archives.
+    * `module_base_path`, specifying the base path for modules with
+      `local_repository` type in the `source.json` file.
 *   `/modules`: A directory containing a subdirectory for each module in this
     registry.
 *   `/modules/$MODULE`: A directory containing a subdirectory for each version
@@ -243,18 +244,29 @@ An index registry must follow the format below:
 *   `/modules/$MODULE/$VERSION`: A directory containing the following files:
     *   `MODULE.bazel`: The `MODULE.bazel` file of this module version.
     *   `source.json`: A JSON file containing information on how to fetch the
-        source of this module version, with the following fields:
-        *   `url`: The URL of the source archive.
-        *   `integrity`: The
-            [Subresource Integrity](https://w3c.github.io/webappsec-subresource-integrity/#integrity-metadata-description){: .external}
-            checksum of the archive.
-        *   `strip_prefix`: A directory prefix to strip when extracting the
-            source archive.
-        *   `patches`: A list of strings, each of which names a patch file to
-            apply to the extracted archive. The patch files are located under
-            the `/modules/$MODULE/$VERSION/patches` directory.
-        *   `patch_strip`: Same as the `--strip` argument of Unix patch.
-    *   `patches/`: An optional directory containing patch files.
+        source of this module version.
+      * The default type is "archive" with the following fields:
+          *   `url`: The URL of the source archive.
+          *   `integrity`: The
+              [Subresource Integrity](https://w3c.github.io/webappsec-subresource-integrity/#integrity-metadata-description){: .external}
+              checksum of the archive.
+          *   `strip_prefix`: A directory prefix to strip when extracting the
+              source archive.
+          *   `patches`: A list of strings, each of which names a patch file to
+              apply to the extracted archive. The patch files are located under
+              the `/modules/$MODULE/$VERSION/patches` directory.
+          *   `patch_strip`: Same as the `--strip` argument of Unix patch.
+      * The type can be changed to use a local path with these fields:
+          *   `type`: `local_path`
+          *   `path`: The local path to the repo, calculated as following:
+              * If path is an absolute path, will be used as it is.
+              * If path is a relative path and `module_base_path` is an absolute path,
+                path is resolved to `<module_base_path>/<path>`
+              * If path and `module_base_path` are both relative paths, path is
+                resolved to `<registry_path>/<module_base_path>/<path>`.
+                Registry must be hosted locally and used by `--registry=file://<registry_path>`.
+                Otherwise, Bazel will throw an error.
+    *   `patches/`: An optional directory containing patch files, only used when `source.json` has "archive" type.
 
 ### Bazel Central Registry {:#bazel-central-registry}
 

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BUILD

@@ -74,6 +74,7 @@ java_library(
         "//src/main/java/com/google/devtools/build/lib/bazel/repository/downloader",
         "//src/main/java/com/google/devtools/build/lib/cmdline",
         "//src/main/java/com/google/devtools/build/lib/events",
+        "//src/main/java/com/google/devtools/build/lib/vfs:pathfragment",
         "//third_party:gson",
         "//third_party:guava",
     ],

src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java

@@ -24,6 +24,7 @@
 import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
 import com.google.devtools.build.lib.cmdline.RepositoryName;
 import com.google.devtools.build.lib.events.ExtendedEventHandler;
+import com.google.devtools.build.lib.vfs.PathFragment;
 import com.google.gson.FieldNamingPolicy;
 import com.google.gson.Gson;
 import com.google.gson.GsonBuilder;
@@ -97,15 +98,18 @@ public Optional<byte[]> getModuleFile(ModuleKey key, ExtendedEventHandler eventH
   /** Represents fields available in {@code bazel_registry.json} for the registry. */
   private static class BazelRegistryJson {
     String[] mirrors;
+    String moduleBasePath;
   }
 
   /** Represents fields available in {@code source.json} for each version of a module. */
   private static class SourceJson {
+    String type = "archive";
     URL url;
     String integrity;
     String stripPrefix;
     Map<String, String> patches;
     int patchStrip;
+    String path;
   }
 
   /**
@@ -147,6 +151,51 @@ public RepoSpec getRepoSpec(
       throw new FileNotFoundException(
           String.format("Module %s's source information not found in registry %s", key, getUrl()));
     }
+
+    String type = sourceJson.get().type;
+    switch (type) {
+      case "archive":
+        return createArchiveRepoSpec(sourceJson, bazelRegistryJson, key, repoName);
+      case "local_path":
+        return createLocalPathRepoSpec(sourceJson, bazelRegistryJson, key, repoName);
+      default:
+        throw new IOException(String.format("Invalid source type for module %s", key));
+    }
+  }
+
+  private RepoSpec createLocalPathRepoSpec(
+      Optional<SourceJson> sourceJson,
+      Optional<BazelRegistryJson> bazelRegistryJson,
+      ModuleKey key,
+      RepositoryName repoName)
+      throws IOException {
+    String path = sourceJson.get().path;
+    if (!PathFragment.isAbsolute(path)) {
+      String moduleBase = bazelRegistryJson.get().moduleBasePath;
+      path = moduleBase + "/" + path;
+      if (!PathFragment.isAbsolute(moduleBase)) {
+        if (uri.getScheme().equals("file")) {
+          path = uri.getPath() + "/" + path;
+        } else {
+          throw new IOException(String.format("Provided non local registry for module %s", key));
+        }
+      }
+    }
+
+    return RepoSpec.builder()
+        .setRuleClassName("local_repository")
+        .setAttributes(
+            ImmutableMap.of(
+                "name", repoName.getName(), "path", PathFragment.create(path).toString()))
+        .build();
+  }
+
+  private RepoSpec createArchiveRepoSpec(
+      Optional<SourceJson> sourceJson,
+      Optional<BazelRegistryJson> bazelRegistryJson,
+      ModuleKey key,
+      RepositoryName repoName)
+      throws IOException {
     URL sourceUrl = sourceJson.get().url;
     if (sourceUrl == null) {
       throw new IOException(String.format("Missing source URL for module %s", key));

src/test/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistryTest.java

@@ -115,7 +115,7 @@ public void testFileUrl() throws Exception {
   }
 
   @Test
-  public void testGetRepoSpec() throws Exception {
+  public void testGetArchiveRepoSpec() throws Exception {
     server.serve(
         "/bazel_registry.json",
         "{",
@@ -183,6 +183,28 @@ public void testGetRepoSpec() throws Exception {
                 .build());
   }
 
+  @Test
+  public void testGetLocalPathRepoSpec() throws Exception {
+    server.serve("/bazel_registry.json", "{", "  \"module_base_path\": \"/hello/foo\"", "}");
+    server.serve(
+        "/modules/foo/1.0/source.json",
+        "{",
+        "  \"type\": \"local_path\",",
+        "  \"path\": \"../bar/project_x\"",
+        "}");
+    server.start();
+
+    Registry registry = registryFactory.getRegistryWithUrl(server.getUrl());
+    assertThat(
+            registry.getRepoSpec(
+                createModuleKey("foo", "1.0"), RepositoryName.create("foorepo"), reporter))
+        .isEqualTo(
+            RepoSpec.builder()
+                .setRuleClassName("local_repository")
+                .setAttributes(ImmutableMap.of("name", "foorepo", "path", "/hello/bar/project_x"))
+                .build());
+  }
+
   @Test
   public void testGetRepoInvalidRegistryJsonSpec() throws Exception {
     server.serve("/bazel_registry.json", "", "", "", "");

src/test/py/bazel/bzlmod/bazel_module_test.py

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+import json
 import os
 import pathlib
 import tempfile
@@ -531,6 +532,46 @@ def testAllowedYankedDepsSuccessMix(self):
                   env_add={'BZLMOD_ALLOW_YANKED_VERSIONS': 'yanked2@1.0'},
                   allow_failure=False)
 
+  def setUpProjectWithLocalRegistryModule(self, dep_name, dep_version,
+                                          module_base_path):
+    bazel_registry = {
+        'module_base_path': module_base_path,
+    }
+    with self.main_registry.root.joinpath('bazel_registry.json').open('w') as f:
+      json.dump(bazel_registry, f, indent=4, sort_keys=True)
+
+    self.main_registry.generateCcSource(dep_name, dep_version)
+    self.main_registry.createLocalPathModule(dep_name, dep_version,
+                                             dep_name + '/' + dep_version)
+
+    self.ScratchFile('main.cc', [
+        '#include "%s.h"' % dep_name,
+        'int main() {',
+        '    hello_%s("main function");' % dep_name,
+        '}',
+    ])
+    self.ScratchFile('MODULE.bazel', [
+        'bazel_dep(name = "%s", version = "%s")' % (dep_name, dep_version),
+    ])
+    self.ScratchFile('BUILD', [
+        'cc_binary(',
+        '  name = "main",',
+        '  srcs = ["main.cc"],',
+        '  deps = ["@%s//:lib_%s"],' % (dep_name, dep_name),
+        ')',
+    ])
+    self.ScratchFile('WORKSPACE', [])
+
+  def testLocalRepoInSourceJsonAbsoluteBasePath(self):
+    self.setUpProjectWithLocalRegistryModule('sss', '1.3',
+                                             str(self.main_registry.projects))
+    _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False)
+    self.assertIn('main function => sss@1.3', stdout)
+
+  def testLocalRepoInSourceJsonRelativeBasePath(self):
+    self.setUpProjectWithLocalRegistryModule('sss', '1.3', 'projects')
+    _, stdout, _ = self.RunBazel(['run', '//:main'], allow_failure=False)
+    self.assertIn('main function => sss@1.3', stdout)
 
 if __name__ == '__main__':
   unittest.main()

src/test/py/bazel/bzlmod/test_utils.py

@@ -267,3 +267,25 @@ def addMetadata(self,
       json.dump(metadata, f, indent=4, sort_keys=True)
 
     return self
+
+  def createLocalPathModule(self, name, version, path):
+    """Add a local module into the registry."""
+    module_dir = self.root.joinpath('modules', name, version)
+    module_dir.mkdir(parents=True, exist_ok=True)
+
+    # Create source.json & copy patch files to the registry
+    source = {
+        'type': 'local_path',
+        'path': path,
+    }
+
+    scratchFile(
+        module_dir.joinpath('MODULE.bazel'), [
+            'module(',
+            '  name = "%s",' % name,
+            '  version = "%s",' % version,
+            ')',
+        ])
+
+    with module_dir.joinpath('source.json').open('w') as f:
+      json.dump(source, f, indent=4, sort_keys=True)