feat: move synthtool templates to library_generation/owlbot (#2443)

This PR transfers the java-specific templates from synthtool to `library_geneation/owlbot` There is a new branch in synthtool (googleapis/synthtool#1923) that has the removed templates. The intention is to keep that synthtool branch as parallel until we fully roll out the hermetic build workflows to both HW libraries and the monorepo. ### Approach We add a list of template exclusions to the configuration yaml and call `java.common_templates` with a custom template path (pointing to our OwlBot Java Postprocessor implementation here in `library_generation` plus the template exclusions found in the yaml. The list of exclusions were obtained from owlbot.py files. Since google-cloud-java has the same exclusions for all libraries, we use a repo-level configuration entry. An example of template excludes is: https://github.com/googleapis/sdk-platform-java/blob/b0a57b70d589e2bdc6e5fcb8cf64d08c3496bc57/java-iam/owlbot.py#L26-L37 ### Different approach possible? We could modify all owlbot.py files to use something other than `java.common_templates`. For example ``` from custom_owlbot import common_templates ... common_templates(excludes=[/*preserved excludes*/]) ``` With such approach, we would not have to parse owlbot.py files and take advantage of the fact it's an executable script. ## Follow up? We probably don't want to call `common_templates` twice, so it may be better to modify owlbot.py files to reference our own implementation instead of synthtool. (This is similar to "Different approach possible?" but it is more of a follow up once the scripts are live). --------- Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com>
googleapis · Feb 8, 2024 · 5c95472 · 5c95472
1 parent dba4c10
commit 5c95472
Show file tree

Hide file tree

Showing 26 changed files with 1,273 additions and 75 deletions.
diff --git a/.github/snippet-bot.yml b/.github/snippet-bot.yml
@@ -4,3 +4,4 @@ ignoreFiles:
   - src/test/**
   - test/**
   - showcase/**
+  - library_generation/owlbot/templates/java_library/samples/install-without-bom/pom.xml
diff --git a/.github/workflows/verify_library_generation.yaml b/.github/workflows/verify_library_generation.yaml
@@ -42,6 +42,7 @@ jobs:
         set -ex
         pushd library_generation
         pip install -r requirements.in
+        pip install .
         popd
     - name: Run integration tests
       shell: bash
@@ -74,6 +75,21 @@ jobs:
         pushd library_generation
         pip install -r requirements.in
         popd
+    - name: install synthtool
+      shell: bash
+      run: |
+        set -ex
+        mkdir -p /tmp/synthtool
+        pushd /tmp/synthtool
+        if [ ! -d "synthtool" ]; then
+          git clone https://github.com/googleapis/synthtool.git
+        fi
+        pushd "synthtool"
+
+        git reset --hard origin/no-java-templates
+
+        python3 -m pip install -e .
+        python3 -m pip install -r requirements.in
     - name: Run shell unit tests
       run: |
         set -x
@@ -108,4 +124,4 @@ jobs:
       run: |
         # exclude generated golden files
         # exclude owlbot until further refaction
-        black --check library_generation --exclude "(library_generation/owlbot)|(library_generation/test/resources/goldens)"
+        black --check library_generation --exclude "(library_generation/test/resources/goldens)"
diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py
@@ -113,6 +113,7 @@ def generate_composed_library(
             config.owlbot_cli_image,
             config.synthtool_commitish,
             str(is_monorepo).lower(),
+            config.path_to_yaml,
         ],
         "Library postprocessing",
     )

diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py
@@ -31,6 +31,8 @@ def __init__(
         googleapis_commitish: str,
         owlbot_cli_image: str,
         synthtool_commitish: str,
+        template_excludes: str,
+        path_to_yaml: str,
         libraries: List[LibraryConfig],
         grpc_version: Optional[str] = None,
         protobuf_version: Optional[str] = None,
@@ -39,6 +41,8 @@ def __init__(
         self.googleapis_commitish = googleapis_commitish
         self.owlbot_cli_image = owlbot_cli_image
         self.synthtool_commitish = synthtool_commitish
+        self.template_excludes = template_excludes
+        self.path_to_yaml = path_to_yaml
         self.libraries = libraries
         self.grpc_version = grpc_version
         self.protobuf_version = protobuf_version
@@ -94,6 +98,8 @@ def from_yaml(path_to_yaml: str):
         googleapis_commitish=__required(config, "googleapis_commitish"),
         owlbot_cli_image=__required(config, "owlbot_cli_image"),
         synthtool_commitish=__required(config, "synthtool_commitish"),
+        template_excludes=__required(config, "template_excludes"),
+        path_to_yaml=path_to_yaml,
         libraries=parsed_libraries,
     )
 

diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh
@@ -26,50 +26,14 @@
 set -ex
 scripts_root=$1
 versions_file=$2
-
-# Runs template and etc in current working directory
-function processModule() {
-  # templates as well as retrieving files from owl-bot-staging
-  echo "Generating templates and retrieving files from owl-bot-staging directory..."
-  if [ -f "owlbot.py" ]
-  then
-    # defaults to run owlbot.py
-    python3 owlbot.py
-  fi
-  echo "...done"
-
-  # write or restore pom.xml files
-  echo "Generating missing pom.xml..."
-  python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
-  echo "...done"
-
-  # write or restore clirr-ignored-differences.xml
-  echo "Generating clirr-ignored-differences.xml..."
-  ${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
-  echo "...done"
-
-  # fix license headers
-  echo "Fixing missing license headers..."
-  python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
-  echo "...done"
-
-  # TODO: re-enable this once we resolve thrashing
-  # restore license headers years
-  # echo "Restoring copyright years..."
-  # /owlbot/bin/restore_license_headers.sh
-  # echo "...done"
-
-  # ensure formatting on all .java files in the repository
-  echo "Reformatting source..."
-  mvn fmt:format -q
-  echo "...done"
-}
+configuration_yaml=$3
 
 # This script can be used to process HW libraries and monorepo
 # (google-cloud-java) libraries, which require a slightly different treatment
 # monorepo folders have an .OwlBot.yaml file in the module folder (e.g.
 # java-asset/.OwlBot.yaml), whereas HW libraries have the yaml in
 # `.github/.OwlBot.yaml`
+monorepo="false"
 if [[ -f "$(pwd)/.OwlBot.yaml" ]]; then
   monorepo="true"
 fi
@@ -80,4 +44,48 @@ if [[ "${monorepo}" == "true" ]]; then
   mv temp owl-bot-staging
 fi
 
-processModule
+
+# Runs template and etc in current working directory
+monorepo=$1
+
+# apply repo templates
+echo "Rendering templates"
+python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${monorepo}"
+
+# templates as well as retrieving files from owl-bot-staging
+echo "Retrieving files from owl-bot-staging directory..."
+if [ -f "owlbot.py" ]
+then
+  # defaults to run owlbot.py
+  python3 owlbot.py
+fi
+echo "...done"
+
+# write or restore pom.xml files
+echo "Generating missing pom.xml..."
+python3 "${scripts_root}/owlbot/src/fix-poms.py" "${versions_file}" "true"
+echo "...done"
+
+# write or restore clirr-ignored-differences.xml
+echo "Generating clirr-ignored-differences.xml..."
+${scripts_root}/owlbot/bin/write_clirr_ignore.sh "${scripts_root}"
+echo "...done"
+
+# fix license headers
+echo "Fixing missing license headers..."
+python3 "${scripts_root}/owlbot/src/fix-license-headers.py"
+echo "...done"
+
+# TODO: re-enable this once we resolve thrashing
+# restore license headers years
+# echo "Restoring copyright years..."
+# /owlbot/bin/restore_license_headers.sh
+# echo "...done"
+
+# ensure formatting on all .java files in the repository
+echo "Reformatting source..."
+mvn fmt:format
+echo "...done"
+
+
+
diff --git a/library_generation/owlbot/src/apply_repo_templates.py b/library_generation/owlbot/src/apply_repo_templates.py
@@ -0,0 +1,42 @@
+"""
+This script parses an owlbot.py file, specifically the call to `java.common_templates` in
+order to extract the excluded files so it can be called with a custom template path
+pointing to the templates hosted in `sdk-platform-java/library_generation/owlbot/templates`.
+Briefly, this wraps the call to synthtool's common templates using a custom template folder.
+"""
+
+import os
+import sys
+from collections.abc import Sequence
+from synthtool.languages.java import common_templates
+from pathlib import Path
+from library_generation.model.generation_config import from_yaml
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+repo_templates_path = os.path.join(script_dir, "..", "templates", "java_library")
+
+
+def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None:
+    config = from_yaml(configuration_yaml_path)
+    print(f"repo_templates_path: {repo_templates_path}")
+    print(f"excludes: {config.template_excludes}")
+    common_templates(
+        excludes=config.template_excludes,
+        template_path=Path(repo_templates_path),
+        monorepo=monorepo,
+    )
+
+
+def main(argv: Sequence[str]) -> None:
+    if len(argv) != 3:
+        raise ValueError(
+            "Usage: python apply-repo-templates.py configuration_yaml_path monorepo"
+        )
+
+    configuration_yaml_path = argv[1]
+    monorepo = argv[2]
+    apply_repo_templates(configuration_yaml_path, monorepo.lower() == "true")
+
+
+if __name__ == "__main__":
+    main(sys.argv)
diff --git a/library_generation/owlbot/src/fix-poms.py b/library_generation/owlbot/src/fix-poms.py
@@ -285,9 +285,11 @@ def update_bom_pom(filename: str, modules: List[module.Module]):
 # https://github.com/googleapis/google-cloud-java/issues/9304
 def __proto_group_id(main_artifact_group_id: str) -> str:
     prefix = "com.google"
-    list_of_group_id = ["com.google.cloud",
-                        "com.google.area120",
-                        "com.google.analytics"]
+    list_of_group_id = [
+        "com.google.cloud",
+        "com.google.area120",
+        "com.google.analytics",
+    ]
     if main_artifact_group_id not in list_of_group_id:
         prefix = main_artifact_group_id
     return f"{prefix}.api.grpc"
@@ -343,11 +345,17 @@ def main(versions_file, monorepo):
     main_module = existing_modules[artifact_id]
 
     # Artifact ID is part of distribution name field in .repo-metadata.json
-    if artifact_id in ["grafeas", "google-cloud-dns",
-                       "google-cloud-notification", "google-iam-policy"]:
+    if artifact_id in [
+        "grafeas",
+        "google-cloud-dns",
+        "google-cloud-notification",
+        "google-iam-policy",
+    ]:
         # There are special libraries that are not automatically generated
-        print(f"Skipping a special case library {artifact_id} that do not have "
-              " the standard module structure.")
+        print(
+            f"Skipping a special case library {artifact_id} that do not have "
+            " the standard module structure."
+        )
         return
 
     parent_artifact_id = f"{artifact_id}-parent"
@@ -383,8 +391,10 @@ def main(versions_file, monorepo):
                 version=main_module.version,
                 release_version=main_module.release_version,
             )
-            if path not in excluded_dependencies_list \
-                    and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -401,8 +411,10 @@ def main(versions_file, monorepo):
                 main_module=main_module,
                 monorepo=monorepo,
             )
-            if path not in excluded_dependencies_list \
-                and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -418,8 +430,10 @@ def main(versions_file, monorepo):
                 version=main_module.version,
                 release_version=main_module.release_version,
             )
-            if path not in excluded_dependencies_list \
-                and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -439,8 +453,10 @@ def main(versions_file, monorepo):
                 proto_module=existing_modules[proto_artifact_id],
                 monorepo=monorepo,
             )
-            if path not in excluded_dependencies_list \
-                and path not in main_module.artifact_id:
+            if (
+                path not in excluded_dependencies_list
+                and path not in main_module.artifact_id
+            ):
                 required_dependencies[path] = module.Module(
                     group_id=__proto_group_id(group_id),
                     artifact_id=path,
@@ -451,13 +467,13 @@ def main(versions_file, monorepo):
         module
         for module in required_dependencies.values()
         if module.artifact_id.startswith("proto-")
-           and module.artifact_id not in parent_artifact_id
+        and module.artifact_id not in parent_artifact_id
     ]
     grpc_modules = [
         module
         for module in required_dependencies.values()
-        if module.artifact_id.startswith("grpc-") \
-           and module.artifact_id not in parent_artifact_id
+        if module.artifact_id.startswith("grpc-")
+        and module.artifact_id not in parent_artifact_id
     ]
     if main_module in grpc_modules or main_module in proto_modules:
         modules = grpc_modules + proto_modules
@@ -489,12 +505,11 @@ def main(versions_file, monorepo):
 
     if os.path.isfile(f"{artifact_id}-bom/pom.xml"):
         print("updating modules in bom pom.xml")
-        if artifact_id+"-bom" not in excluded_poms_list:
+        if artifact_id + "-bom" not in excluded_poms_list:
             update_bom_pom(f"{artifact_id}-bom/pom.xml", modules)
-    elif artifact_id+"-bom" not in excluded_poms_list:
+    elif artifact_id + "-bom" not in excluded_poms_list:
         print("creating missing bom pom.xml")
-        monorepo_version = __get_monorepo_version(versions_file) \
-            if monorepo else ""
+        monorepo_version = __get_monorepo_version(versions_file) if monorepo else ""
         templates.render(
             template_name="bom_pom.xml.j2",
             output_name=f"{artifact_id}-bom/pom.xml",
@@ -503,16 +518,15 @@ def main(versions_file, monorepo):
             modules=modules,
             main_module=main_module,
             monorepo=monorepo,
-            monorepo_version=monorepo_version
+            monorepo_version=monorepo_version,
         )
 
     if os.path.isfile("pom.xml"):
         print("updating modules in parent pom.xml")
         update_parent_pom("pom.xml", modules)
     else:
         print("creating missing parent pom.xml")
-        monorepo_version = __get_monorepo_version(versions_file) \
-            if monorepo else ""
+        monorepo_version = __get_monorepo_version(versions_file) if monorepo else ""
         templates.render(
             template_name="parent_pom.xml.j2",
             output_name="./pom.xml",
@@ -521,7 +535,7 @@ def main(versions_file, monorepo):
             main_module=main_module,
             name=name,
             monorepo=monorepo,
-            monorepo_version=monorepo_version
+            monorepo_version=monorepo_version,
         )
 
     print(f"updating modules in {versions_file}")
@@ -537,13 +551,15 @@ def main(versions_file, monorepo):
                 release_version=main_module.release_version,
             )
     templates.render(
-        template_name="versions.txt.j2", output_name=versions_file, modules=existing_modules.values(),
+        template_name="versions.txt.j2",
+        output_name=versions_file,
+        modules=existing_modules.values(),
     )
 
 
 if __name__ == "__main__":
     versions_file = sys.argv[1]
     monorepo = sys.argv[2]
-    if monorepo == 'true':
+    if monorepo == "true":
         monorepo = True
     main(versions_file, monorepo)