diff --git a/library_generation/README.md b/library_generation/README.md index 1a9e77893f..efb42b2d48 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -91,14 +91,22 @@ Use `--grpc_version` to specify the value. Note that if specified, the version should be compatible with gapic-generator-java. +### gapic_additional_protos (optional) +Additional protos that pass to the generator. +The default value is `google/cloud/common_resources.proto`. + +Use `--gapic_additional_protos` to specify the value. + ### transport (optional) -One of GAPIC options passed to the generator. The value is either `grpc` or `grpc+rest`. +One of GAPIC options passed to the generator. +The value is either `grpc` or `grpc+rest`. The default value is `grpc`. Use `--transport` to specify the value. ### rest_numeric_enums (optional) -One of GAPIC options passed to the generator. The value is either `true` or `false`. +One of GAPIC options passed to the generator. +The value is either `true` or `false`. The default value is `true`. Use `--rest_numeric_enums` to specify the value. @@ -121,6 +129,7 @@ library_generation/generate_library.sh \ --gapic_generator_version 2.24.0 \ --protobuf_version 23.2 \ --grpc_version 1.55.1 \ +--gapic_additional_protos "google/cloud/common_resources.proto google/cloud/location/locations.proto" \ --transport grpc+rest \ --rest_numeric_enums true \ --include_samples true diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index 812d1b1359..9863a6170e 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -29,6 +29,10 @@ case $key in grpc_version="$2" shift ;; + --gapic_additional_protos) + gapic_additional_protos="$2" + shift + ;; --transport) transport="$2" shift @@ -56,7 +60,6 @@ done script_dir=$(dirname "$(readlink -f "$0")") source "${script_dir}"/utilities.sh output_folder="$(get_output_folder)" -# source utility functions if [ -z "${protobuf_version}" ]; then protobuf_version=$(get_protobuf_version "${gapic_generator_version}") @@ -66,6 +69,10 @@ if [ -z "${grpc_version}" ]; then grpc_version=$(get_grpc_version "${gapic_generator_version}") fi +if [ -z "${gapic_additional_protos}" ]; then + gapic_additional_protos="google/cloud/common_resources.proto" +fi + if [ -z "${transport}" ]; then transport="grpc" fi @@ -117,7 +124,7 @@ fi "--plugin=protoc-gen-java_gapic=${script_dir}/gapic-generator-java-wrapper" \ "--java_gapic_out=metadata:${destination_path}/java_gapic_srcjar_raw.srcjar.zip" \ "--java_gapic_opt=$(get_gapic_opts)" \ -${proto_files} $(search_additional_protos) +${proto_files} ${gapic_additional_protos} unzip -o -q "${destination_path}/java_gapic_srcjar_raw.srcjar.zip" -d "${destination_path}" # Sync'\''d to the output file name in Writer.java. diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 7b42402837..96429c6e43 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -68,6 +68,7 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do # parse GAPIC options from proto_path/BUILD.bazel pushd "${output_folder}" proto_build_file_path="${proto_path}/BUILD.bazel" + gapic_additional_protos=$(get_gapic_additional_protos_from_BUILD "${proto_build_file_path}") transport=$(get_transport_from_BUILD "${proto_build_file_path}") rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path}") include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}") @@ -81,6 +82,7 @@ grep -v '^ *#' < "${proto_path_list}" | while IFS= read -r line; do --gapic_generator_version "${gapic_generator_version}" \ --protobuf_version "${protobuf_version}" \ --grpc_version "${grpc_version}" \ + --gapic_additional_protos "${gapic_additional_protos}" \ --transport "${transport}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index b945ffffb1..4af6da36ec 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -41,40 +41,6 @@ get_protobuf_version_failed_with_invalid_generator_version_test() { assertEquals 1 $((res)) } -search_additional_protos_common_resources_test() { - local proto_path="${script_dir}/resources/search_additional_proto/common_resources" - local addition_protos - addition_protos=$(search_additional_protos) - assertEquals "google/cloud/common_resources.proto" "${addition_protos}" -} - -search_additional_protos_iam_test() { - local proto_path="${script_dir}/resources/search_additional_protos/iam" - local addition_protos - addition_protos=$(search_additional_protos) - assertEquals \ - "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto" \ - "${addition_protos}" -} - -search_additional_protos_location_test() { - local proto_path="${script_dir}/resources/search_additional_protos/location" - local addition_protos - addition_protos=$(search_additional_protos) - assertEquals \ - "google/cloud/common_resources.proto google/cloud/location/locations.proto" \ - "${addition_protos}" -} - -search_additional_protos_iam_location_test() { - local proto_path="${script_dir}/resources/search_additional_protos/iam_location" - local addition_protos - addition_protos=$(search_additional_protos) - assertEquals \ - "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" \ - "${addition_protos}" -} - get_gapic_opts_with_rest_test() { local proto_path="${script_dir}/resources/gapic_options" local transport="grpc" @@ -227,6 +193,34 @@ generate_library_failed_with_invalid_grpc_version() { cleanup "${destination}" } +get_gapic_additional_protos_from_BUILD_common_resources_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_common_resources.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto" "${addition_protos}" +} + +get_gapic_additional_protos_from_BUILD_iam_policy_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_policy.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto" "${addition_protos}" +} + +get_gapic_additional_protos_from_BUILD_locations_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_locations.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto google/cloud/location/locations.proto" "${addition_protos}" +} + +get_gapic_additional_protos_from_BUILD_iam_locations_test() { + local proto_path="${script_dir}/resources/search_additional_protos/BUILD_iam_locations.bazel" + local addition_protos + addition_protos=$(get_gapic_additional_protos_from_BUILD "${proto_path}") + assertEquals "google/cloud/common_resources.proto google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" "${addition_protos}" +} + get_transport_from_BUILD_grpc_rest_test() { local build_file="${script_dir}/resources/misc/BUILD_grpc_rest.bazel" local transport @@ -316,10 +310,6 @@ test_list=( get_grpc_version_failed_with_invalid_generator_version_test get_protobuf_version_succeed_with_valid_generator_version_test get_protobuf_version_failed_with_invalid_generator_version_test - search_additional_protos_common_resources_test - search_additional_protos_iam_test - search_additional_protos_location_test - search_additional_protos_iam_location_test get_gapic_opts_with_rest_test get_gapic_opts_without_rest_test remove_grpc_version_test @@ -336,6 +326,10 @@ test_list=( generate_library_failed_with_invalid_generator_version generate_library_failed_with_invalid_protobuf_version generate_library_failed_with_invalid_grpc_version + get_gapic_additional_protos_from_BUILD_common_resources_test + get_gapic_additional_protos_from_BUILD_iam_policy_test + get_gapic_additional_protos_from_BUILD_locations_test + get_gapic_additional_protos_from_BUILD_iam_locations_test get_transport_from_BUILD_grpc_rest_test get_transport_from_BUILD_grpc_test get_transport_from_BUILD_rest_test diff --git a/library_generation/test/resources/search_additional_protos/BUILD_common_resources.bazel b/library_generation/test/resources/search_additional_protos/BUILD_common_resources.bazel new file mode 100644 index 0000000000..45e3987adb --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_common_resources.bazel @@ -0,0 +1,6 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_locations.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_locations.bazel new file mode 100644 index 0000000000..e8241995e2 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_locations.bazel @@ -0,0 +1,8 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_iam_policy.bazel b/library_generation/test/resources/search_additional_protos/BUILD_iam_policy.bazel new file mode 100644 index 0000000000..81064a7eb4 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_iam_policy.bazel @@ -0,0 +1,7 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + "//google/iam/v1:iam_policy_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/BUILD_locations.bazel b/library_generation/test/resources/search_additional_protos/BUILD_locations.bazel new file mode 100644 index 0000000000..07faa4ac95 --- /dev/null +++ b/library_generation/test/resources/search_additional_protos/BUILD_locations.bazel @@ -0,0 +1,7 @@ +# this file is only used in testing `get_gapic_additional_protos_from_BUILD` in utilities.sh + +proto_library_with_info( + deps = [ + "//google/cloud/location:location_proto", + ] +) \ No newline at end of file diff --git a/library_generation/test/resources/search_additional_protos/common_resources/search_additional_protos_common_resources_test.yaml b/library_generation/test/resources/search_additional_protos/common_resources/search_additional_protos_common_resources_test.yaml deleted file mode 100644 index fc5c8bfca5..0000000000 --- a/library_generation/test/resources/search_additional_protos/common_resources/search_additional_protos_common_resources_test.yaml +++ /dev/null @@ -1,10 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.monitoring.v3.AlertPolicyService -- name: google.monitoring.v3.GroupService -- name: google.monitoring.v3.MetricService -- name: google.monitoring.v3.NotificationChannelService -- name: google.monitoring.v3.QueryService -- name: google.monitoring.v3.ServiceMonitoringService -- name: google.monitoring.v3.SnoozeService -- name: google.monitoring.v3.UptimeCheckService diff --git a/library_generation/test/resources/search_additional_protos/iam/search_additional_protos_iam_test.yaml b/library_generation/test/resources/search_additional_protos/iam/search_additional_protos_iam_test.yaml deleted file mode 100644 index cc946f1e11..0000000000 --- a/library_generation/test/resources/search_additional_protos/iam/search_additional_protos_iam_test.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.iam.v1.IAMPolicy -- name: google.pubsub.v1.Publisher -- name: google.pubsub.v1.SchemaService -- name: google.pubsub.v1.Subscriber diff --git a/library_generation/test/resources/search_additional_protos/iam_location/search_additional_protos_iam_location_test.yaml b/library_generation/test/resources/search_additional_protos/iam_location/search_additional_protos_iam_location_test.yaml deleted file mode 100644 index 0789b073e4..0000000000 --- a/library_generation/test/resources/search_additional_protos/iam_location/search_additional_protos_iam_location_test.yaml +++ /dev/null @@ -1,6 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.cloud.alloydb.v1.AlloyDBAdmin -- name: google.cloud.location.Locations -- name: google.iam.v1.IAMPolicy -- name: google.longrunning.Operations diff --git a/library_generation/test/resources/search_additional_protos/location/search_additional_protos_location_test.yaml b/library_generation/test/resources/search_additional_protos/location/search_additional_protos_location_test.yaml deleted file mode 100644 index 73abf4dc7c..0000000000 --- a/library_generation/test/resources/search_additional_protos/location/search_additional_protos_location_test.yaml +++ /dev/null @@ -1,5 +0,0 @@ -# this file is only used in testing `search_additional_protos` in utilities.sh -apis: -- name: google.cloud.location.Locations -- name: google.firestore.v1.Firestore -- name: google.longrunning.Operations diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 4ccdc42f32..2c90cb1554 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -18,12 +18,38 @@ __get_config_from_BUILD() { if_match=$5 result="${default}" - if grep -A 15 "${rule}" "${build_file}" | grep -q "${pattern}"; then + if grep -A 20 "${rule}" "${build_file}" | grep -q "${pattern}"; then result="${if_match}" fi echo "${result}" } +__get_iam_policy_from_BUILD() { + local build_file=$1 + local contains_iam_policy + contains_iam_policy=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library_with_info(" \ + "//google/iam/v1:iam_policy_proto" \ + "false" \ + "true" + ) + echo "${contains_iam_policy}" +} + +__get_locations_from_BUILD() { + local build_file=$1 + local contains_locations + contains_locations=$(__get_config_from_BUILD \ + "${build_file}" \ + "proto_library_with_info(" \ + "//google/cloud/location:location_proto" \ + "false" \ + "true" + ) + echo "${contains_locations}" +} + # define utility functions extract_folder_name() { local destination_path=$1 @@ -69,33 +95,6 @@ unzip_src_files() { rm -r -f "${destination_path}/${category}-${folder_name}/src/main/java/META-INF" } -find_additional_protos_in_yaml() { - local pattern=$1 - local find_result - find_result=$(grep --include=\*.yaml -rw "${proto_path}" -e "${pattern}") - if [ -n "${find_result}" ]; then - echo "${find_result}" - fi -} - -# Apart from proto files in proto_path, additional protos are needed in order -# to generate GAPIC client libraries. -# In most cases, these protos should be within google/ directory, which is -# pulled from googleapis as a prerequisite. -# Search additional protos in .yaml files. -search_additional_protos() { - additional_protos="google/cloud/common_resources.proto" # used by every library - iam_policy=$(find_additional_protos_in_yaml "name: '*google.iam.v1.IAMPolicy'*") - if [ -n "$iam_policy" ]; then - additional_protos="$additional_protos google/iam/v1/iam_policy.proto" - fi - locations=$(find_additional_protos_in_yaml "name: '*google.cloud.location.Locations'*") - if [ -n "${locations}" ]; then - additional_protos="$additional_protos google/cloud/location/locations.proto" - fi - echo "${additional_protos}" -} - # get gapic options from .yaml and .json files from proto_path. get_gapic_opts() { local gapic_config @@ -253,6 +252,23 @@ get_version_from_WORKSPACE() { echo "${version}" } +# Apart from proto files in proto_path, additional protos are needed in order +# to generate GAPIC client libraries. +# In most cases, these protos should be within google/ directory, which is +# pulled from googleapis as a prerequisite. +# Get additional protos in BUILD.bazel. +get_gapic_additional_protos_from_BUILD() { + local build_file=$1 + local gapic_additional_protos="google/cloud/common_resources.proto" + if [[ $(__get_iam_policy_from_BUILD "${build_file}") == "true" ]]; then + gapic_additional_protos="${gapic_additional_protos} google/iam/v1/iam_policy.proto" + fi + if [[ $(__get_locations_from_BUILD "${build_file}") == "true" ]]; then + gapic_additional_protos="${gapic_additional_protos} google/cloud/location/locations.proto" + fi + echo "${gapic_additional_protos}" +} + get_transport_from_BUILD() { local build_file=$1 local transport diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index 6619f5801d..1b25e3ae68 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -39,6 +39,7 @@ fi ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator-java") +gapic_additional_protos="google/iam/v1/iam_policy.proto google/cloud/location/locations.proto" rest_numeric_enums="false" transport="grpc+rest" include_samples="false" @@ -49,6 +50,7 @@ bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ --proto_path "schema/google/showcase/v1beta1" \ --destination_path "showcase-output" \ --gapic_generator_version "${ggj_version}" \ + --gapic_additional_protos "${gapic_additional_protos}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" \ --transport "${transport}"