feat: generate showcase without post-processing (#1935)

* feat: add * generate gapic and proto folder * refactor utilities * add an action to verify * checkout googleapis-gen * setup repo name * add commit hash of googleapis-gen * change secret * change token * change to git clone * change user name * add input list * include resources folder in main * remove grpc version in `*ServiceGrpc.java` * change destination path * compare generation result with googleapis-gen * fix type in diff command * checkout repo using checkout action * checkout repos as nested repo * sparse checkout googleapis * Revert "sparse checkout googleapis" This reverts commit 3d612f8. * change library * change step name * add a readme * make grpc version optional * make protobuf version optional * checkout master branch, rather than a commit hash * allow snapshot version of generator * download snapshot of generator parent pom * update README * chore: add test script for showcase this produces various differences in the output. Mainly additional protos not being detected, a mismatch between proto-gapic-showcase-v1beta1/src/proto (arbitrary path) * chore: fix additional protos logic * download generator and grpc using mvn * change error message * add maven central mirror * add comments in utilities * add comments * add an integration test * chore: modify showcase source proto folder structure * fail fast if no file is found * chore: add showcase integration test * chore: add action to test on showase * chore: add diff check for showcase IT * do not delete google/ * chore: change file mod in protos * chore: fix diff comparison * chore: remove unwanted binary * get protobuf version from WORKSPACE * add instructions on download `google/` from googleapis * add comments * chore: final adjustments for showcase testing * chore: allow workflow dispatch on workflow * chore: correct yaml definition * chore: correct yaml definition ii * chore: fix util import * update description of `destination_path` * chore: correct version parsing * chore: correct showcase version * chore: try java 11 * update comments * chore: infer grpc_version from parent pom * download dependencies using `curl` * increase download time * remove comment * generate components for showcase * use script in maven profile * correct compilation components * explicit output folder path check * use poms to resolve versions * migrate showcase scripts to showcase folder * remove bazel-based update script * finalize migration for generation * fix generation * change verification profile for gapic-showcase * remove unused functions in verify.sh * remove unused CI component * remove showcase from readme * remove unused logic * script comments and cleanup * improve comment in generate_component * remove unused bazel files * remove .gitignore in favor of cleanup * bash format * transfer profiles to showcase parent * fix parent pom script call * fix wildcard for source replacement * fix proto golden test, add util script * install xmllint * sudo for installing xmlint * correct xmllint package * Update library_generation/utilities.sh Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> * restore gh action * move convenience functions to utilities file * remove duplicate file\ * fix proto path * use `sed` and `cut` to extract showcase version * add unit tests for new functions * add test for versions --------- Co-authored-by: JoeWang1127 <joewa@google.com> Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com>
googleapis · Sep 12, 2023 · 7218d80 · 7218d80
1 parent 41f6ef6
commit 7218d80
Show file tree

Hide file tree

Showing 24 changed files with 467 additions and 377 deletions.
diff --git a/library_generation/README.md b/library_generation/README.md
@@ -125,15 +125,3 @@ library_generation/generate_library.sh \
 --include_samples true
 ```
 
-## An example to generate showcase client
-```
-library_generation/generate_library.sh \
--p schema/google/showcase/v1beta1 \ # google/ should be in library_generation/.
--d output \
---gapic_generator_version 2.24.0 \
---protobuf_version 23.2 \
---grpc_version 1.55.1 \
---transport grpc+rest \
---rest_numeric_enums false \
---include_samples false
-```
diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 
 set -eo pipefail
+set -x
 
 # parse input parameters
 while [[ $# -gt 0 ]]; do
@@ -157,3 +158,4 @@ done
 #####################################################
 cd "${destination_path}"
 rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar
+set +x
diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh
@@ -42,34 +42,13 @@ esac
 shift # past argument or value
 done
 
-get_version_from_WORKSPACE() {
-  local version_key_word=$1
-  local workspace=$2
-  local delimiter=$3
-  local version
-  version="$(grep -m 1 "${version_key_word}"  "${workspace}" | sed 's/\"\(.*\)\".*/\1/' | cut -d "${delimiter}" -f2 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')"
-  echo "${version}"
-}
-
-sparse_clone() {
-  local repo_url=$1
-  local paths=$2
-  local clone_dir
-  clone_dir=$(basename "${repo_url%.*}")
-  rm -rf "${clone_dir}"
-  git clone -n --depth=1 --filter=tree:0 "${repo_url}"
-  cd "${clone_dir}"
-  git sparse-checkout set --no-cone ${paths}
-  git checkout
-  cd ..
-}
-
 script_dir=$(dirname "$(readlink -f "$0")")
+source "${script_dir}/../utilities.sh"
 library_generation_dir="${script_dir}"/..
 cd "${library_generation_dir}"
 # checkout the master branch of googleapis/google (proto files) and WORKSPACE
 echo "Checking out googlapis repository..."
-sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto"
+sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/iam/v1 google/type google/longrunning"
 cd googleapis
 # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE
 gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=")
@@ -79,18 +58,25 @@ echo "The version of protobuf is ${protobuf_version}"
 grpc_version=$(get_version_from_WORKSPACE "_grpc_version" WORKSPACE "=")
 echo "The version of protoc-gen-grpc-java plugin is ${gapic_generator_version}."
 # parse GAPIC options from proto_path/BUILD.bazel
-transport="grpc"
-if grep -A 15 "java_gapic_library(" "${proto_path}/BUILD.bazel" | grep -q "grpc+rest"; then
-  transport="grpc+rest"
-fi
-rest_numeric_enums="true"
-if grep -A 15 "java_gapic_library(" "${proto_path}/BUILD.bazel" | grep -q "rest_numeric_enums = False"; then
-  rest_numeric_enums="false"
-fi
-include_samples="false"
-if grep -A 15 "java_gapic_assembly_gradle_pkg(" "${proto_path}/BUILD.bazel" | grep -q "include_samples = True"; then
-  include_samples="true"
-fi
+proto_build_file_path="${proto_path}/BUILD.bazel"
+transport=$(get_config_from_BUILD \
+  "${proto_build_file_path}" \
+  "java_gapic_library(" \
+  "grpc+rest" \
+  "grpc"
+)
+rest_numeric_enums=$(get_config_from_BUILD \
+  "${proto_build_file_path}" \
+  "java_gapic_library(" \
+  "rest_numeric_enums = False" \
+  "true"
+)
+include_samples=$(get_config_from_BUILD \
+  "${proto_build_file_path}" \
+  "java_gapic_assembly_gradle_pkg(" \
+  "include_samples = True" \
+  "false"
+)
 echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}."
 os_architecture="linux-x86_64"
 if [[ "$os_type" == *"macos"* ]]; then

diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh
@@ -219,6 +219,48 @@ download_grpc_plugin_failed_with_invalid_arch_test() {
   __assertEquals 1 $((res))
 }
 
+get_config_from_valid_BUILD_test() {
+  build_file="${script_dir}/resources/misc/TESTBUILD.bazel"
+  rule="java_gapic_library("
+  # the pattern we expect to find in the BUILD file
+  pattern_should_match="name"
+  # the pattern that we should not find in the BUILD file
+  pattern_should_not_match="should-not-match"
+  # default value if the pattern was not found
+  if_matched_return="got-a-match"
+  if_not_matched_return="no-match"
+  pattern_matched_result=$(get_config_from_BUILD \
+    "${build_file}" \
+    "${rule}" \
+    "${pattern_should_match}" \
+    "${if_not_matched_return}" \
+    "${if_matched_return}"
+  )
+  __assertEquals "${pattern_matched_result}" "${if_matched_return}"
+  pattern_not_matched_result=$(get_config_from_BUILD \
+    "${build_file}" \
+    "${rule}" \
+    "${pattern_should_not_match}" \
+    "${if_not_matched_return}" \
+    "${if_matched_return}"
+  )
+  __assertEquals "${pattern_not_matched_result}" "${if_not_matched_return}"
+}
+
+get_version_from_valid_WORKSPACE_test() {
+  workspace_file="${script_dir}/resources/misc/TESTWORKSPACE"
+  obtained_ggj_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" "${workspace_file}")
+  __assertEquals '2.25.1-SNAPSHOT' "${obtained_ggj_version}"
+}
+
+get_version_from_valid_versions_txt_test() {
+  versions_file="${script_dir}/resources/misc/testversions.txt"
+  obtained_ggj_version=$(get_version_from_versions_txt "${versions_file}" "gapic-generator-java")
+  __assertEquals '2.25.1-SNAPSHOT' "${obtained_ggj_version}"
+  obtained_gax_version=$(get_version_from_versions_txt "${versions_file}" "gax")
+  __assertEquals '2.33.1-SNAPSHOT' "${obtained_gax_version}"
+}
+
 # Execute tests.
 # One line per test.
 test_list=(
@@ -244,6 +286,9 @@ test_list=(
   download_grpc_plugin_succeed_with_valid_version_macos_test
   download_grpc_plugin_failed_with_invalid_version_linux_test
   download_grpc_plugin_failed_with_invalid_arch_test
+  get_config_from_valid_BUILD_test
+  get_version_from_valid_WORKSPACE_test
+  get_version_from_valid_versions_txt_test
 )
 
 for ut in "${test_list[@]}"; do

diff --git a/showcase/BUILD.bazel → ...ation/test/resources/misc/TESTBUILD.bazel b/showcase/BUILD.bazel → ...ation/test/resources/misc/TESTBUILD.bazel
@@ -1,5 +1,4 @@
-# Note: The commented section is the framework for gapic-showcase-extended
-# See gapic-showcase-extended folder for more info
+# Test BUILD file, copied from sdk-platform-java/showcase/BUILD.bazel
 
 load(
     "//rules_java_gapic:java_gapic.bzl",

diff --git a/library_generation/test/resources/misc/TESTWORKSPACE b/library_generation/test/resources/misc/TESTWORKSPACE
@@ -0,0 +1,133 @@
+# test workspace file obtained from sdk-platform-java
+
+workspace(name = "gapic_generator_java")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+# gax-java and its transitive dependencies must be imported before
+# gapic-generator-java dependencies to match the order in googleapis repository,
+# which in its turn, prioritizes actual generated clients runtime dependencies
+# over the generator dependencies.
+local_repository(
+   name = "com_google_api_gax_java",
+   path = "gax-java",
+)
+
+load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties")
+
+com_google_api_gax_java_properties(
+    name = "com_google_api_gax_java_properties",
+    file = "@com_google_api_gax_java//:dependencies.properties",
+)
+
+load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories")
+
+com_google_api_gax_java_repositories()
+
+_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d"
+
+http_archive(
+    name = "com_google_googleapis",
+    strip_prefix = "googleapis-%s" % _googleapis_commit,
+    urls = [
+        "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit,
+    ],
+)
+
+# protobuf
+RULES_JVM_EXTERNAL_TAG = "4.5"
+
+RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6"
+
+http_archive(
+    name = "rules_jvm_external",
+    sha256 = RULES_JVM_EXTERNAL_SHA,
+    strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG,
+    url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG,
+)
+
+load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps")
+
+rules_jvm_external_deps()
+
+load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup")
+
+rules_jvm_external_setup()
+
+load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps")
+load("@rules_jvm_external//:defs.bzl", "maven_install")
+
+maven_install(
+    artifacts = PROTOBUF_MAVEN_ARTIFACTS,
+    repositories = ["https://repo.maven.apache.org/maven2/"],
+)
+
+_gapic_generator_java_version = "2.25.1-SNAPSHOT"  # {x-version-update:gapic-generator-java:current}
+
+maven_install(
+    artifacts = [
+        "com.google.api:gapic-generator-java:" + _gapic_generator_java_version,
+    ],
+    fail_on_missing_checksum = False,
+    repositories = [
+        "m2Local",
+        "https://repo.maven.apache.org/maven2/",
+    ],
+)
+
+protobuf_deps()
+
+# Bazel rules.
+_rules_gapic_version = "0.5.5"
+
+http_archive(
+    name = "rules_gapic",
+    strip_prefix = "rules_gapic-%s" % _rules_gapic_version,
+    urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version],
+)
+
+# Java dependencies.
+load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language")
+
+switched_rules_by_language(
+    name = "com_google_googleapis_imports",
+    gapic = True,
+    grpc = True,
+    java = True,
+)
+
+load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories")
+
+grpc_java_repositories()
+
+_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3"
+
+http_archive(
+    name = "com_google_disco_to_proto3_converter",
+    strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit,
+    urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit],
+)
+
+# Showcase
+_showcase_version = "0.28.2"
+
+http_archive(
+    name = "com_google_gapic_showcase",
+    strip_prefix = "gapic-showcase-%s" % _showcase_version,
+    urls = [
+        "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version,
+    ],
+)
+
+http_archive(
+    name = "rules_pkg",
+    sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2",
+    urls = [
+        "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
+        "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz",
+    ],
+)
+
+load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies")
+
+rules_pkg_dependencies()
diff --git a/library_generation/test/resources/misc/testversions.txt b/library_generation/test/resources/misc/testversions.txt
@@ -0,0 +1,9 @@
+# test versions.txt obtained from sdk-platform-java
+
+gapic-generator-java:2.25.0:2.25.1-SNAPSHOT
+api-common:2.16.0:2.16.1-SNAPSHOT
+gax:2.33.0:2.33.1-SNAPSHOT
+gax-grpc:2.34.0:2.33.1-SNAPSHOT
+gax-httpjson:0.118.0:0.118.1-SNAPSHOT
+proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT
+grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT