diff --git a/library_generation/README.md b/library_generation/README.md index 0b8db04757..8279b32282 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -125,15 +125,3 @@ library_generation/generate_library.sh \ --include_samples true ``` -## An example to generate showcase client -``` -library_generation/generate_library.sh \ --p schema/google/showcase/v1beta1 \ # google/ should be in library_generation/. --d output \ ---gapic_generator_version 2.24.0 \ ---protobuf_version 23.2 \ ---grpc_version 1.55.1 \ ---transport grpc+rest \ ---rest_numeric_enums false \ ---include_samples false -``` diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index e691fdc8af..e20d9f1856 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -1,6 +1,7 @@ #!/usr/bin/env bash set -eo pipefail +set -x # parse input parameters while [[ $# -gt 0 ]]; do @@ -157,3 +158,4 @@ done ##################################################### cd "${destination_path}" rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar +set +x diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index c9402a5188..1c437d328a 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -42,34 +42,13 @@ esac shift # past argument or value done -get_version_from_WORKSPACE() { - local version_key_word=$1 - local workspace=$2 - local delimiter=$3 - local version - version="$(grep -m 1 "${version_key_word}" "${workspace}" | sed 's/\"\(.*\)\".*/\1/' | cut -d "${delimiter}" -f2 | sed 's/^[[:space:]]*//;s/[[:space:]]*$//')" - echo "${version}" -} - -sparse_clone() { - local repo_url=$1 - local paths=$2 - local clone_dir - clone_dir=$(basename "${repo_url%.*}") - rm -rf "${clone_dir}" - git clone -n --depth=1 --filter=tree:0 "${repo_url}" - cd "${clone_dir}" - git sparse-checkout set --no-cone ${paths} - git checkout - cd .. -} - script_dir=$(dirname "$(readlink -f "$0")") +source "${script_dir}/../utilities.sh" library_generation_dir="${script_dir}"/.. cd "${library_generation_dir}" # checkout the master branch of googleapis/google (proto files) and WORKSPACE echo "Checking out googlapis repository..." -sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto" +sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/iam/v1 google/type google/longrunning" cd googleapis # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=") @@ -79,18 +58,25 @@ echo "The version of protobuf is ${protobuf_version}" grpc_version=$(get_version_from_WORKSPACE "_grpc_version" WORKSPACE "=") echo "The version of protoc-gen-grpc-java plugin is ${gapic_generator_version}." # parse GAPIC options from proto_path/BUILD.bazel -transport="grpc" -if grep -A 15 "java_gapic_library(" "${proto_path}/BUILD.bazel" | grep -q "grpc+rest"; then - transport="grpc+rest" -fi -rest_numeric_enums="true" -if grep -A 15 "java_gapic_library(" "${proto_path}/BUILD.bazel" | grep -q "rest_numeric_enums = False"; then - rest_numeric_enums="false" -fi -include_samples="false" -if grep -A 15 "java_gapic_assembly_gradle_pkg(" "${proto_path}/BUILD.bazel" | grep -q "include_samples = True"; then - include_samples="true" -fi +proto_build_file_path="${proto_path}/BUILD.bazel" +transport=$(get_config_from_BUILD \ + "${proto_build_file_path}" \ + "java_gapic_library(" \ + "grpc+rest" \ + "grpc" +) +rest_numeric_enums=$(get_config_from_BUILD \ + "${proto_build_file_path}" \ + "java_gapic_library(" \ + "rest_numeric_enums = False" \ + "true" +) +include_samples=$(get_config_from_BUILD \ + "${proto_build_file_path}" \ + "java_gapic_assembly_gradle_pkg(" \ + "include_samples = True" \ + "false" +) echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}." os_architecture="linux-x86_64" if [[ "$os_type" == *"macos"* ]]; then diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index 84fb248aa3..84e3f11fd0 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -219,6 +219,48 @@ download_grpc_plugin_failed_with_invalid_arch_test() { __assertEquals 1 $((res)) } +get_config_from_valid_BUILD_test() { + build_file="${script_dir}/resources/misc/TESTBUILD.bazel" + rule="java_gapic_library(" + # the pattern we expect to find in the BUILD file + pattern_should_match="name" + # the pattern that we should not find in the BUILD file + pattern_should_not_match="should-not-match" + # default value if the pattern was not found + if_matched_return="got-a-match" + if_not_matched_return="no-match" + pattern_matched_result=$(get_config_from_BUILD \ + "${build_file}" \ + "${rule}" \ + "${pattern_should_match}" \ + "${if_not_matched_return}" \ + "${if_matched_return}" + ) + __assertEquals "${pattern_matched_result}" "${if_matched_return}" + pattern_not_matched_result=$(get_config_from_BUILD \ + "${build_file}" \ + "${rule}" \ + "${pattern_should_not_match}" \ + "${if_not_matched_return}" \ + "${if_matched_return}" + ) + __assertEquals "${pattern_not_matched_result}" "${if_not_matched_return}" +} + +get_version_from_valid_WORKSPACE_test() { + workspace_file="${script_dir}/resources/misc/TESTWORKSPACE" + obtained_ggj_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" "${workspace_file}") + __assertEquals '2.25.1-SNAPSHOT' "${obtained_ggj_version}" +} + +get_version_from_valid_versions_txt_test() { + versions_file="${script_dir}/resources/misc/testversions.txt" + obtained_ggj_version=$(get_version_from_versions_txt "${versions_file}" "gapic-generator-java") + __assertEquals '2.25.1-SNAPSHOT' "${obtained_ggj_version}" + obtained_gax_version=$(get_version_from_versions_txt "${versions_file}" "gax") + __assertEquals '2.33.1-SNAPSHOT' "${obtained_gax_version}" +} + # Execute tests. # One line per test. test_list=( @@ -244,6 +286,9 @@ test_list=( download_grpc_plugin_succeed_with_valid_version_macos_test download_grpc_plugin_failed_with_invalid_version_linux_test download_grpc_plugin_failed_with_invalid_arch_test + get_config_from_valid_BUILD_test + get_version_from_valid_WORKSPACE_test + get_version_from_valid_versions_txt_test ) for ut in "${test_list[@]}"; do diff --git a/showcase/BUILD.bazel b/library_generation/test/resources/misc/TESTBUILD.bazel similarity index 97% rename from showcase/BUILD.bazel rename to library_generation/test/resources/misc/TESTBUILD.bazel index 14435f6903..21d38a35b9 100644 --- a/showcase/BUILD.bazel +++ b/library_generation/test/resources/misc/TESTBUILD.bazel @@ -1,5 +1,4 @@ -# Note: The commented section is the framework for gapic-showcase-extended -# See gapic-showcase-extended folder for more info +# Test BUILD file, copied from sdk-platform-java/showcase/BUILD.bazel load( "//rules_java_gapic:java_gapic.bzl", diff --git a/library_generation/test/resources/misc/TESTWORKSPACE b/library_generation/test/resources/misc/TESTWORKSPACE new file mode 100644 index 0000000000..60b3036d9d --- /dev/null +++ b/library_generation/test/resources/misc/TESTWORKSPACE @@ -0,0 +1,133 @@ +# test workspace file obtained from sdk-platform-java + +workspace(name = "gapic_generator_java") + +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +# gax-java and its transitive dependencies must be imported before +# gapic-generator-java dependencies to match the order in googleapis repository, +# which in its turn, prioritizes actual generated clients runtime dependencies +# over the generator dependencies. +local_repository( + name = "com_google_api_gax_java", + path = "gax-java", +) + +load("@com_google_api_gax_java//:repository_rules.bzl", "com_google_api_gax_java_properties") + +com_google_api_gax_java_properties( + name = "com_google_api_gax_java_properties", + file = "@com_google_api_gax_java//:dependencies.properties", +) + +load("@com_google_api_gax_java//:repositories.bzl", "com_google_api_gax_java_repositories") + +com_google_api_gax_java_repositories() + +_googleapis_commit = "7438480b2a1bc6371d748e974f7a3647f90c4e8d" + +http_archive( + name = "com_google_googleapis", + strip_prefix = "googleapis-%s" % _googleapis_commit, + urls = [ + "https://github.com/googleapis/googleapis/archive/%s.zip" % _googleapis_commit, + ], +) + +# protobuf +RULES_JVM_EXTERNAL_TAG = "4.5" + +RULES_JVM_EXTERNAL_SHA = "b17d7388feb9bfa7f2fa09031b32707df529f26c91ab9e5d909eb1676badd9a6" + +http_archive( + name = "rules_jvm_external", + sha256 = RULES_JVM_EXTERNAL_SHA, + strip_prefix = "rules_jvm_external-%s" % RULES_JVM_EXTERNAL_TAG, + url = "https://github.com/bazelbuild/rules_jvm_external/archive/%s.zip" % RULES_JVM_EXTERNAL_TAG, +) + +load("@rules_jvm_external//:repositories.bzl", "rules_jvm_external_deps") + +rules_jvm_external_deps() + +load("@rules_jvm_external//:setup.bzl", "rules_jvm_external_setup") + +rules_jvm_external_setup() + +load("@com_google_protobuf//:protobuf_deps.bzl", "PROTOBUF_MAVEN_ARTIFACTS", "protobuf_deps") +load("@rules_jvm_external//:defs.bzl", "maven_install") + +maven_install( + artifacts = PROTOBUF_MAVEN_ARTIFACTS, + repositories = ["https://repo.maven.apache.org/maven2/"], +) + +_gapic_generator_java_version = "2.25.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} + +maven_install( + artifacts = [ + "com.google.api:gapic-generator-java:" + _gapic_generator_java_version, + ], + fail_on_missing_checksum = False, + repositories = [ + "m2Local", + "https://repo.maven.apache.org/maven2/", + ], +) + +protobuf_deps() + +# Bazel rules. +_rules_gapic_version = "0.5.5" + +http_archive( + name = "rules_gapic", + strip_prefix = "rules_gapic-%s" % _rules_gapic_version, + urls = ["https://github.com/googleapis/rules_gapic/archive/v%s.tar.gz" % _rules_gapic_version], +) + +# Java dependencies. +load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") + +switched_rules_by_language( + name = "com_google_googleapis_imports", + gapic = True, + grpc = True, + java = True, +) + +load("@io_grpc_grpc_java//:repositories.bzl", "grpc_java_repositories") + +grpc_java_repositories() + +_disco_to_proto3_converter_commit = "ce8d8732120cdfb5bf4847c3238b5be8acde87e3" + +http_archive( + name = "com_google_disco_to_proto3_converter", + strip_prefix = "disco-to-proto3-converter-%s" % _disco_to_proto3_converter_commit, + urls = ["https://github.com/googleapis/disco-to-proto3-converter/archive/%s.zip" % _disco_to_proto3_converter_commit], +) + +# Showcase +_showcase_version = "0.28.2" + +http_archive( + name = "com_google_gapic_showcase", + strip_prefix = "gapic-showcase-%s" % _showcase_version, + urls = [ + "https://github.com/googleapis/gapic-showcase/archive/refs/tags/v%s.zip" % _showcase_version, + ], +) + +http_archive( + name = "rules_pkg", + sha256 = "8a298e832762eda1830597d64fe7db58178aa84cd5926d76d5b744d6558941c2", + urls = [ + "https://mirror.bazel.build/github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", + "https://github.com/bazelbuild/rules_pkg/releases/download/0.7.0/rules_pkg-0.7.0.tar.gz", + ], +) + +load("@rules_pkg//:deps.bzl", "rules_pkg_dependencies") + +rules_pkg_dependencies() diff --git a/library_generation/test/resources/misc/testversions.txt b/library_generation/test/resources/misc/testversions.txt new file mode 100644 index 0000000000..e4258504e4 --- /dev/null +++ b/library_generation/test/resources/misc/testversions.txt @@ -0,0 +1,9 @@ +# test versions.txt obtained from sdk-platform-java + +gapic-generator-java:2.25.0:2.25.1-SNAPSHOT +api-common:2.16.0:2.16.1-SNAPSHOT +gax:2.33.0:2.33.1-SNAPSHOT +gax-grpc:2.34.0:2.33.1-SNAPSHOT +gax-httpjson:0.118.0:0.118.1-SNAPSHOT +proto-google-common-protos:2.24.0:2.24.1-SNAPSHOT +grpc-google-common-protos:2.24.0:2.24.1-SNAPSHOT diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index bddd51e1df..d45af802c4 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -63,16 +63,14 @@ find_additional_protos_in_yaml() { # pulled from googleapis as a prerequisite. # Search additional protos in .yaml files. search_additional_protos() { - local additional_protos="google/cloud/common_resources.proto" # used by every library - local iam_policy - local locations - iam_policy=$(find_additional_protos_in_yaml "name: google.iam.v1.IAMPolicy") - if [ -n "${iam_policy}" ]; then - additional_protos="${additional_protos} google/iam/v1/iam_policy.proto" + additional_protos="google/cloud/common_resources.proto" # used by every library + iam_policy=$(find_additional_protos_in_yaml "name: '*google.iam.v1.IAMPolicy'*") + if [ -n "$iam_policy" ]; then + additional_protos="$additional_protos google/iam/v1/iam_policy.proto" fi - locations=$(find_additional_protos_in_yaml "name: google.cloud.location.Locations") + locations=$(find_additional_protos_in_yaml "name: '*google.cloud.location.Locations'*") if [ -n "${locations}" ]; then - additional_protos="${additional_protos} google/cloud/location/locations.proto" + additional_protos="$additional_protos google/cloud/location/locations.proto" fi echo "${additional_protos}" } @@ -216,3 +214,67 @@ download_fail() { >&2 echo "Fail to download ${artifact} from ${repo} repository. Please install ${artifact} first if you want to download a SNAPSHOT." exit 1 } + +# Obtains a version from a bazel WORKSPACE file +# +# versions look like "_ggj_version="1.2.3" +# It will return 1.2.3 for such example +get_version_from_WORKSPACE() { + version_key_word=$1 + workspace=$2 + version=$(\ + cat "$workspace" |\ + grep "$version_key_word" |\ + head -n 1 |\ + sed 's/\(.*\) = "\(.*\)"\(.*\)/\2/' |\ + sed 's/[a-zA-Z-]*//' + ) + echo "$version" +} + +# Used to obtain configuration values from a bazel BUILD file +# +# inspects a $build_file for a certain $rule (e.g. java_gapic_library). If the +# first 15 lines after the declaration of the rule contain $pattern, then +# it will return $if_match if $pattern is found, otherwise $default +get_config_from_BUILD() { + build_file=$1 + rule=$2 + pattern=$3 + default=$4 + if_match=$5 + + result="$default" + if grep -A 15 "$rule" "$build_file" | grep -q "$pattern"; then + result="$if_match" + fi + echo "$result" + +} + +# Convenience function to clone only the necessary folders from a git repository +sparse_clone() { + repo_url=$1 + paths=$2 + commitish=$3 + clone_dir=$(basename "${repo_url%.*}") + rm -rf "$clone_dir" + git clone -n --depth=1 --no-single-branch --filter=tree:0 "$repo_url" + cd "$clone_dir" + if [ ! -z $commitish ]; then + git checkout $commitish + fi + git sparse-checkout set --no-cone $paths + git checkout + cd .. +} + +# takes a versions.txt file and returns its version +get_version_from_versions_txt() { + versions=$1 + key=$2 + version=$(cat "$versions" | grep "$key:" | cut -d: -f3) # 3rd field is snapshot + echo $version +} + + diff --git a/showcase/gapic-showcase/pom.xml b/showcase/gapic-showcase/pom.xml index 84947ccf66..5eab85a38d 100644 --- a/showcase/gapic-showcase/pom.xml +++ b/showcase/gapic-showcase/pom.xml @@ -22,67 +22,6 @@ 0.28.2 - - - enable-golden-tests - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - - verify-showcase - - exec - - test - - bazel - - run - --experimental_proto_descriptor_sets_include_source_info - //showcase:verify_gapic - - - - - - - - - - update - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - - verify-showcase - - exec - - generate-sources - - bazelisk - - run - --experimental_proto_descriptor_sets_include_source_info - //showcase:update_gapic - - - - - - - - - - diff --git a/showcase/grpc-gapic-showcase-v1beta1/pom.xml b/showcase/grpc-gapic-showcase-v1beta1/pom.xml index af1bf89017..50b89a1e81 100644 --- a/showcase/grpc-gapic-showcase-v1beta1/pom.xml +++ b/showcase/grpc-gapic-showcase-v1beta1/pom.xml @@ -17,67 +17,6 @@ true - - - enable-golden-tests - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - - verify-showcase - - exec - - test - - bazel - - run - --experimental_proto_descriptor_sets_include_source_info - //showcase:verify_grpc - - - - - - - - - - update - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - - verify-showcase - - exec - - generate-sources - - bazelisk - - run - --experimental_proto_descriptor_sets_include_source_info - //showcase:update_grpc - - - - - - - - - - io.grpc diff --git a/showcase/pom.xml b/showcase/pom.xml index 2d88469198..14fb9091d1 100644 --- a/showcase/pom.xml +++ b/showcase/pom.xml @@ -125,5 +125,63 @@ + + enable-golden-tests + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + false + + + verify-showcase + + exec + + test + + bash + + scripts/verify.sh + + + + + + + + + + update + + + + org.codehaus.mojo + exec-maven-plugin + 3.1.0 + false + + + verify-showcase + + exec + + generate-sources + + bash + + scripts/generate_components.sh + true + true + + + + + + + + diff --git a/showcase/proto-gapic-showcase-v1beta1/pom.xml b/showcase/proto-gapic-showcase-v1beta1/pom.xml index 2c9438793f..e4d74f6a84 100644 --- a/showcase/proto-gapic-showcase-v1beta1/pom.xml +++ b/showcase/proto-gapic-showcase-v1beta1/pom.xml @@ -17,67 +17,6 @@ true - - - enable-golden-tests - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - - verify-showcase - - exec - - test - - bazelisk - - run - --experimental_proto_descriptor_sets_include_source_info - //showcase:verify_proto - - - - - - - - - - update - - - - org.codehaus.mojo - exec-maven-plugin - 3.1.0 - - - verify-showcase - - exec - - generate-sources - - bazel - - run - --experimental_proto_descriptor_sets_include_source_info - //showcase:update_proto - - - - - - - - - - com.google.protobuf diff --git a/showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/compliance.proto b/showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/compliance.proto old mode 100755 new mode 100644 similarity index 100% rename from showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/compliance.proto rename to showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/compliance.proto diff --git a/showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/echo.proto b/showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/echo.proto old mode 100755 new mode 100644 similarity index 100% rename from showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/echo.proto rename to showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/echo.proto diff --git a/showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/identity.proto b/showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/identity.proto old mode 100755 new mode 100644 similarity index 100% rename from showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/identity.proto rename to showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/identity.proto diff --git a/showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/messaging.proto b/showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/messaging.proto old mode 100755 new mode 100644 similarity index 100% rename from showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/messaging.proto rename to showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/messaging.proto diff --git a/showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/sequence.proto b/showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/sequence.proto old mode 100755 new mode 100644 similarity index 100% rename from showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/sequence.proto rename to showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/sequence.proto diff --git a/showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/testing.proto b/showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/testing.proto old mode 100755 new mode 100644 similarity index 100% rename from showcase/proto-gapic-showcase-v1beta1/src/main/proto/external/com_google_gapic_showcase/schema/google/showcase/v1beta1/testing.proto rename to showcase/proto-gapic-showcase-v1beta1/src/main/proto/schema/google/showcase/v1beta1/testing.proto diff --git a/showcase/scripts/BUILD.bazel b/showcase/scripts/BUILD.bazel deleted file mode 100644 index 504a2a6198..0000000000 --- a/showcase/scripts/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//showcase:update"]) - -exports_files([ - "update.sh", - "verify.sh", -]) diff --git a/showcase/scripts/generate_components.sh b/showcase/scripts/generate_components.sh new file mode 100755 index 0000000000..740d8b3caf --- /dev/null +++ b/showcase/scripts/generate_components.sh @@ -0,0 +1,42 @@ +#!/bin/bash +# +# Generates the three components of the showcase library: +# If $1 (replace) is "true" then the output of the generation will be written as +# the new showcase source code +# If $2 (perform_cleanup) is "true" then the generated library will be deleted +# along with the necessary tools to generate it +set -xe +replace=$1 +perform_cleanup=$2 +if [ -z "${replace}" ]; then + replace="false" +fi +readonly SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/showcase_utilities.sh" + +if [ ! -d "${SCRIPT_DIR}/showcase-output" ]; then + bash "${SCRIPT_DIR}/generate_showcase.sh" +fi + +if [ "${replace}" == "true" ]; then + showcase_folder="${SCRIPT_DIR}/.." + + # replace contents of gapic library + gapic_component_dir="${showcase_folder}/gapic-showcase/src/main" + rm -rdf "${gapic_component_dir}/*" + cp -r "${SCRIPT_DIR}/showcase-output/gapic-showcase-output/src/main"/* "${gapic_component_dir}/" + + # replace contents of proto library + proto_component_dir="${showcase_folder}/proto-gapic-showcase-v1beta1/src/main" + rm -rdf "${proto_component_dir}/*" + cp -r "${SCRIPT_DIR}/showcase-output/proto-showcase-output/src/main"/* "${proto_component_dir}/" + + # replace contents of grpc library + grpc_component_dir="${showcase_folder}/grpc-gapic-showcase-v1beta1/src/main" + rm -rdf "${grpc_component_dir}/*" + cp -r "${SCRIPT_DIR}/showcase-output/grpc-showcase-output/src/main"/* "${grpc_component_dir}/" +fi + +if [ "${perform_cleanup}" == 'true' ];then + cleanup "${SCRIPT_DIR}" +fi diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh new file mode 100755 index 0000000000..4d25ca3155 --- /dev/null +++ b/showcase/scripts/generate_showcase.sh @@ -0,0 +1,60 @@ +#!/bin/bash +# Generates showcase without post processing (i.e. raw gapic, grpc and proto +# libraries). It will compute the showcase version from +# `sdk-platform-java/showcase/gapic-showcase/pom.xml`. The generator version is +# inferred from versions.txt +set -ex + +readonly SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +lib_gen_scripts_dir="${SCRIPT_DIR}/../../library_generation/" +source "${lib_gen_scripts_dir}/utilities.sh" +readonly perform_cleanup=$1 + +cd "${SCRIPT_DIR}" + +# clone gapic-showcase +if [ ! -d schema ]; then + if [ -d gapic-showcase ]; then + rm -rdf gapic-showcase + fi + # looks at sdk-platform-java/showcase/gapic-showcase/pom.xml to extract the + # version of gapic-showcase + # see https://github.com/googleapis/gapic-showcase/releases + showcase_version=$(grep -e '' "${SCRIPT_DIR}/../gapic-showcase/pom.xml" | cut -d'>' -f2 | cut -d'<' -f1) + sparse_clone https://github.com/googleapis/gapic-showcase.git "schema/google/showcase/v1beta1" "v${showcase_version}" + cd gapic-showcase + mv schema .. + cd .. + rm -rdf gapic-showcase +fi +if [ ! -d google ];then + if [ -d googleapis ]; then + rm -rdf googleapis + fi + sparse_clone https://github.com/googleapis/googleapis.git "WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/longrunning google/iam/v1 google/cloud/location google/type" + mv googleapis/google . + rm -rdf googleapis +fi + + +ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-java") +rest_numeric_enums="false" +transport="grpc+rest" +include_samples="false" +rm -rdf showcase-output +mkdir showcase-output +set +e +bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ + --proto_path "schema/google/showcase/v1beta1" \ + --destination_path "showcase-output" \ + --gapic_generator_version "${ggj_version}" \ + --rest_numeric_enums "${rest_numeric_enums}" \ + --include_samples "${include_samples}" \ + --transport "${transport}" &> out + +exit_code=$? +if [ "${exit_code}" -ne 0 ]; then + rm -rdf showcase-output + exit "${exit_code}" +fi +set +x diff --git a/showcase/scripts/showcase_utilities.sh b/showcase/scripts/showcase_utilities.sh new file mode 100644 index 0000000000..8db80248ca --- /dev/null +++ b/showcase/scripts/showcase_utilities.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +function cleanup { + script_dir=$1 + cd "${script_dir}" + rm -rdf gapic-generator-java* google schema protobuf-* protoc-gen-grpc-java* showcase-output out +} diff --git a/showcase/scripts/update.sh b/showcase/scripts/update.sh deleted file mode 100755 index f7b7e4ee7a..0000000000 --- a/showcase/scripts/update.sh +++ /dev/null @@ -1,69 +0,0 @@ -#!/bin/sh -# This script is executed by ../BUILD.bazel as a final post-generation step. - -set -o errexit -BAZEL_ROOT=$PWD - -clear_existing() { - cd "$BUILD_WORKSPACE_DIRECTORY/showcase/$1" - find . -name '*.java' -not -path '*/it/*' -delete - find . -name 'gapic_metadata.json' -delete - cd - -} -create_unpack_dir() { - cd "$BAZEL_ROOT" - rm -rf "$1" - mkdir "$1" - cd "$1" -} -delete_unneeded() { - find . -name '.DS_Store' -delete - find . -name 'PlaceholderFile.java' -delete - find . -type d -empty -delete -} - -case $1 in - proto) - PROTO_PROJECT_DIR=proto-gapic-showcase-v1beta1 - PROTO_ARCHIVE=$(find . -name 'proto-google-cloud-showcase-v1beta1-java.tar.gz') - clear_existing $PROTO_PROJECT_DIR - create_unpack_dir proto_unpacked - PROTO_UNPACK_DIR=$PWD - - tar -xzf "$BAZEL_ROOT/$PROTO_ARCHIVE" - delete_unneeded - cd "$BUILD_WORKSPACE_DIRECTORY/showcase/$PROTO_PROJECT_DIR" - mkdir -p ./src - cp -r "$PROTO_UNPACK_DIR"/proto-google-cloud-showcase-v1beta1-java/src/* ./src - ;; - - grpc) - GRPC_PROJECT_DIR=grpc-gapic-showcase-v1beta1 - GRPC_JAR=$(find . -name 'libshowcase_java_grpc-src.jar') -# GRPC_JAR_EXTENDED=$(find . -name 'libshowcase_java_grpc_extended-src.jar') - clear_existing $GRPC_PROJECT_DIR - create_unpack_dir grpc_unpacked - GRPC_UNPACK_DIR=$PWD - - jar xf "$BAZEL_ROOT/$GRPC_JAR" -# jar xf "$BAZEL_ROOT/$GRPC_JAR_EXTENDED" - cd "$BUILD_WORKSPACE_DIRECTORY/showcase/$GRPC_PROJECT_DIR" - mkdir -p ./src/main/java/com - cp -r "$GRPC_UNPACK_DIR"/com/* ./src/main/java/com - ;; - - gapic) - GAPIC_PROJECT_DIR=gapic-showcase - GAPIC_JAR=$(find . -name 'showcase_java_gapic_srcjar_raw.srcjar') - clear_existing $GAPIC_PROJECT_DIR - create_unpack_dir gapic_unpacked - GAPIC_UNPACK_DIR=$PWD - - unzip -q -c "$BAZEL_ROOT/$GAPIC_JAR" temp-codegen.srcjar | jar x - cd "$BUILD_WORKSPACE_DIRECTORY/showcase/$GAPIC_PROJECT_DIR" - cp -r "$GAPIC_UNPACK_DIR"/* ./ - ;; -esac - -cd "${BUILD_WORKSPACE_DIRECTORY}/showcase" -delete_unneeded diff --git a/showcase/scripts/verify.sh b/showcase/scripts/verify.sh index 8aed7020c8..e3d4c61392 100755 --- a/showcase/scripts/verify.sh +++ b/showcase/scripts/verify.sh @@ -1,70 +1,27 @@ #!/bin/sh -# This script is executed by ../BUILD.bazel as a final post-generation step. +# This script generates showcase in a temporary/untracked folder and compares +# its contents with the actual showcase libraries. set -o errexit -BAZEL_ROOT=$PWD -SHOWCASE_DIR="$BUILD_WORKSPACE_DIRECTORY/showcase" +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +source "${SCRIPT_DIR}/showcase_utilities.sh" +SHOWCASE_DIR="${SCRIPT_DIR}/.." -clear_existing() { - cd "$SHOWCASE_DIR/$1" - find . -name '*.java' -not -path '*/it/*' -delete - find . -name 'gapic_metadata.json' -delete - find . -name 'reflect-config.json' -delete - cd - -} -create_unpack_dir() { - cd "$BAZEL_ROOT" - rm -rf "$1" - mkdir "$1" - cd "$1" -} -delete_unneeded() { - find . -name '.DS_Store' -delete - find . -name 'PlaceholderFile.java' -delete - find . -type d -empty -delete -} +# generate sources +readonly REPLACE_SOURCE='false' +readonly PERFORM_CLEANUP='false' +bash "${SCRIPT_DIR}/generate_components.sh" "${REPLACE_SOURCE}" "${PERFORM_CLEANUP}" -case $1 in - proto) - PROTO_PROJECT_DIR=proto-gapic-showcase-v1beta1 - PROTO_ARCHIVE_NAME=proto-google-cloud-showcase-v1beta1-java - PROTO_ARCHIVE=$(find . -name "$PROTO_ARCHIVE_NAME.tar.gz") - create_unpack_dir proto_unpacked - PROTO_UNPACK_DIR=$PWD +# compare with proto library +PROTO_PROJECT_DIR='proto-gapic-showcase-v1beta1' +diff -ru "${SHOWCASE_DIR}/${PROTO_PROJECT_DIR}/src" "${SCRIPT_DIR}/showcase-output/proto-showcase-output/src" - tar -xzf "$BAZEL_ROOT/$PROTO_ARCHIVE" - delete_unneeded - diff -ru "$SHOWCASE_DIR/$PROTO_PROJECT_DIR"/src/main/java "$PROTO_UNPACK_DIR/$PROTO_ARCHIVE_NAME"/src/main/java - ;; +# compare with grpc library +GRPC_PROJECT_DIR='grpc-gapic-showcase-v1beta1' +diff -ru "${SHOWCASE_DIR}/${GRPC_PROJECT_DIR}/src/main/java/com" "${SCRIPT_DIR}/showcase-output/grpc-showcase-output/src/main/java/com" - grpc) - GRPC_PROJECT_DIR=grpc-gapic-showcase-v1beta1 - GRPC_JAR=$(find . -name 'libshowcase_java_grpc-src.jar') -# GRPC_JAR_EXTENDED=$(find . -name 'libshowcase_java_grpc_extended-src.jar') - create_unpack_dir grpc_unpacked - GRPC_UNPACK_DIR=$PWD +# compare with gapic library +GAPIC_PROJECT_DIR='gapic-showcase' +diff -ru "${SHOWCASE_DIR}/${GAPIC_PROJECT_DIR}/src/main/java" "${SCRIPT_DIR}/showcase-output/gapic-showcase-output/src/main/java" - jar xf "$BAZEL_ROOT/$GRPC_JAR" -# jar xf "$BAZEL_ROOT/$GRPC_JAR_EXTENDED" - delete_unneeded - diff -ru "$SHOWCASE_DIR/$GRPC_PROJECT_DIR"/src/main/java/com "$GRPC_UNPACK_DIR"/com - ;; - - gapic) - GAPIC_PROJECT_DIR=gapic-showcase - GAPIC_JAR=$(find . -name 'showcase_java_gapic_srcjar_raw.srcjar') - create_unpack_dir gapic_unpacked - GAPIC_UNPACK_DIR=$PWD - - unzip -q -c "$BAZEL_ROOT/$GAPIC_JAR" temp-codegen.srcjar | jar x - delete_unneeded - - # "diff --exclude" matches against file and directory basenames, not pathnames. - # To exclude "test/resources" without excluding "main/resources", we copy the showcase project - # to a temporary directory, and delete what we want to exclude. - cp -r "$SHOWCASE_DIR/$GAPIC_PROJECT_DIR"/src "$GAPIC_UNPACK_DIR"/golden - rm -r "$GAPIC_UNPACK_DIR"/golden/test/java/com/google/showcase/v1beta1/it - rm -r "$GAPIC_UNPACK_DIR"/golden/test/resources - diff -ru "$GAPIC_UNPACK_DIR"/golden "$GAPIC_UNPACK_DIR"/src --exclude=*.iml - ;; -esac +cleanup $SCRIPT_DIR