From f5efb0edb097980db95832469188c754ec3f1779 Mon Sep 17 00:00:00 2001 From: Diego Alonso Marquez Palacios Date: Thu, 14 Sep 2023 17:47:40 -0400 Subject: [PATCH] feat: hermetic build scripts to use a single output/generation folder (#1987) * move generation tools and generated source to single folder * script fixes * capture exit code from failing script * remove accidental file * wip: fix showcase test * post rebase fixes * adjust showcase failure handling * update readme * fix output path in IT * download googleapis in output folder * Update library_generation/README.md Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> * remove unrelated os detection function * use pushd instead of cd in gen script output cleanup * correct destination_path documentation * Update library_generation/README.md Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> * fix pushd in generate_library.sh * ignore output folder * Update library_generation/README.md Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> * Update library_generation/README.md Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> * spit output to stdout * centralize output folder computation * label popped folders * prevent wrong version unit test from exiting the test suite * resolve output dir using pwd --------- Co-authored-by: Joe Wang <106995533+JoeWang1127@users.noreply.github.com> --- .gitignore | 7 ++++- library_generation/README.md | 23 ++++++++-------- library_generation/generate_library.sh | 13 +++++++--- .../test/generate_library_integration_test.sh | 21 +++++++++------ .../test/generate_library_unit_tests.sh | 17 +++++------- library_generation/utilities.sh | 16 +++++++++--- showcase/scripts/generate_showcase.sh | 13 +++++----- showcase/scripts/showcase_utilities.sh | 2 +- showcase/scripts/verify.sh | 26 +++++++++++++------ 9 files changed, 86 insertions(+), 52 deletions(-) diff --git a/.gitignore b/.gitignore index 07604728ff..f98cac050a 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,9 @@ target/ # Vscode Settings .vscode/settings.json -*.iml \ No newline at end of file +*.iml + +# library generation +output/ +library_generation/output/ +showcase/scripts/output/ diff --git a/library_generation/README.md b/library_generation/README.md index c16d92de0a..1a9e77893f 100644 --- a/library_generation/README.md +++ b/library_generation/README.md @@ -7,17 +7,18 @@ The script, `generate_library.sh`, allows you to generate a GAPIC client library Use Linux environment and install java runtime environment (8 or above). ## Prerequisite -Protos referenced by protos in `proto_path` (see `proto_path` below) should be copied to the current -working directory (refers as `$cwd`, a directory contains `generate_library.sh`). +Protos referenced by protos in `proto_path` (see `proto_path` below) should be copied to an `output` +directory located in the working directory, referred as `$cwd` +(for example, `library_generation/output` if planning to call from the same folder). The directory structure should be the same as import statements in protos. -For example, we want to generate from `folder1/folder2/protoA`, so `proto_path` -should be set to `folder1/folder2` (a relative path from `$cwd`). +For example, you want to generate from `folder1/folder2/protoA`, so `proto_path` +should be set to `folder1/folder2` (a relative path from `output`). protoA imports protoB as `folder3/folder4/protoB`, then there should -be `folder3/folder4` (containing protoB) in `$cwd`. +be `folder3/folder4` (containing protoB) in `output`. In order to generate a GAPIC library, you need to pull `google/` from [googleapis](https://github.com/googleapis/googleapis) -and put it into `$cwd` since protos in `google/` are likely referenced by +and put it into `output` since protos in `google/` are likely referenced by protos from which the library are generated. ## Parameters to run `generate_library.sh` @@ -25,19 +26,19 @@ protos from which the library are generated. You need to run the script with the following parameters. ### proto_path -A directory in `$cwd` and copy proto files into it. -The absolute path of `proto_path` is `$cwd/$proto_path`. +A directory in `$cwd/output` and copy proto files into it. +The absolute path of `proto_path` is `$cwd/output/$proto_path`. Use `-p` or `--proto_path` to specify the value. ### destination_path -A directory within `$cwd`. +A directory within `$cwd/output`. This is the path in which the generated library will reside. -The absolute path of `destination_path` is `$cwd/$destination_path`. +The absolute path of `destination_path` is `$cwd/output/$destination_path`. Use `-d` or `--destination_path` to specify the value. -Note that you do not need to create `$detination_path` beforehand. +Note that you do not need to create `$destination_path` beforehand. The directory structure of the generated library is ``` diff --git a/library_generation/generate_library.sh b/library_generation/generate_library.sh index d48f19fae5..812d1b1359 100755 --- a/library_generation/generate_library.sh +++ b/library_generation/generate_library.sh @@ -54,8 +54,9 @@ shift # past argument or value done script_dir=$(dirname "$(readlink -f "$0")") -# source utility functions source "${script_dir}"/utilities.sh +output_folder="$(get_output_folder)" +# source utility functions if [ -z "${protobuf_version}" ]; then protobuf_version=$(get_protobuf_version "${gapic_generator_version}") @@ -81,15 +82,17 @@ if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -mkdir -p "${destination_path}" + +mkdir -p "${output_folder}/${destination_path}" ##################### Section 0 ##################### # prepare tooling ##################################################### # the order of services entries in gapic_metadata.json is relevant to the # order of proto file, sort the proto files with respect to their name to # get a fixed order. -proto_files=$(find "${proto_path}" -type f -name "*.proto" | sort) folder_name=$(extract_folder_name "${destination_path}") +pushd "${output_folder}" +proto_files=$(find "${proto_path}" -type f -name "*.proto" | sort) # download gapic-generator-java, protobuf and grpc plugin. download_tools "${gapic_generator_version}" "${protobuf_version}" "${grpc_version}" "${os_architecture}" ##################### Section 1 ##################### @@ -156,9 +159,11 @@ for proto_src in ${proto_files}; do mkdir -p "${destination_path}/proto-${folder_name}/src/main/proto" rsync -R "${proto_src}" "${destination_path}/proto-${folder_name}/src/main/proto" done +popd # output_folder ##################### Section 4 ##################### # rm tar files ##################################################### -cd "${destination_path}" +pushd "${output_folder}/${destination_path}" rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_proto.jar temp-codegen.srcjar +popd set +x diff --git a/library_generation/test/generate_library_integration_test.sh b/library_generation/test/generate_library_integration_test.sh index 391b46c243..0e37577768 100755 --- a/library_generation/test/generate_library_integration_test.sh +++ b/library_generation/test/generate_library_integration_test.sh @@ -14,6 +14,10 @@ set -xeo pipefail # defaults googleapis_gen_url="git@github.com:googleapis/googleapis-gen.git" +script_dir=$(dirname "$(readlink -f "$0")") +source "${script_dir}/../utilities.sh" +library_generation_dir="${script_dir}"/.. +output_folder="$(get_output_folder)" while [[ $# -gt 0 ]]; do key="$1" @@ -38,14 +42,13 @@ esac shift # past argument or value done -script_dir=$(dirname "$(readlink -f "$0")") -source "${script_dir}/../utilities.sh" -library_generation_dir="${script_dir}"/.. -cd "${library_generation_dir}" +mkdir -p "${output_folder}" +pushd "${output_folder}" # checkout the master branch of googleapis/google (proto files) and WORKSPACE echo "Checking out googlapis repository..." sparse_clone https://github.com/googleapis/googleapis.git "${proto_path} WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/iam/v1 google/type google/longrunning" -cd googleapis +pushd googleapis +cp -r google "${output_folder}" # parse version of gapic-generator-java, protobuf and grpc from WORKSPACE gapic_generator_version=$(get_version_from_WORKSPACE "_gapic_generator_java_version" WORKSPACE "=") echo "The version of gapic-generator-java is ${gapic_generator_version}." @@ -60,6 +63,8 @@ rest_numeric_enums=$(get_rest_numeric_enums_from_BUILD "${proto_build_file_path} include_samples=$(get_include_samples_from_BUILD "${proto_build_file_path}") echo "GAPIC options are transport=${transport}, rest_numeric_enums=${rest_numeric_enums}, include_samples=${include_samples}." # generate GAPIC client library +popd +popd echo "Generating library from ${proto_path}, to ${destination_path}..." "${library_generation_dir}"/generate_library.sh \ -p "${proto_path}" \ @@ -74,11 +79,12 @@ echo "Generating library from ${proto_path}, to ${destination_path}..." echo "Generate library finished." echo "Checking out googleapis-gen repository..." +pushd "${output_folder}" sparse_clone "${googleapis_gen_url}" "${proto_path}" echo "Compare generation result..." RESULT=0 -diff -r "googleapis-gen/${proto_path}/${destination_path}" "${destination_path}" -x "*gradle*" || RESULT=$? +diff -r "googleapis-gen/${proto_path}/${destination_path}" "${output_folder}/${destination_path}" -x "*gradle*" || RESULT=$? if [ ${RESULT} == 0 ] ; then echo "SUCCESS: Comparison finished, no difference is found." @@ -86,7 +92,6 @@ else echo "FAILURE: Differences found." fi -cd .. -rm -rf googleapis +popd # output_folder exit ${RESULT} diff --git a/library_generation/test/generate_library_unit_tests.sh b/library_generation/test/generate_library_unit_tests.sh index b74b0f6215..b945ffffb1 100755 --- a/library_generation/test/generate_library_unit_tests.sh +++ b/library_generation/test/generate_library_unit_tests.sh @@ -181,15 +181,14 @@ generate_library_failed_with_invalid_generator_version() { local destination="google-cloud-alloydb-v1-java" local res=0 cd "${script_dir}/resources" - $("${script_dir}"/../generate_library.sh \ + bash "${script_dir}"/../generate_library.sh \ -p google/cloud/alloydb/v1 \ -d ../"${destination}" \ --gapic_generator_version 1.99.0 \ --protobuf_version 23.2 \ --grpc_version 1.55.1 \ --transport grpc+rest \ - --rest_numeric_enums true \ - --os_architecture "$(__get_os_architecture)") || res=$? + --rest_numeric_enums true || res=$? assertEquals 1 $((res)) # still need to clean up potential downloaded tooling. cleanup "${destination}" @@ -199,15 +198,14 @@ generate_library_failed_with_invalid_protobuf_version() { local destination="google-cloud-alloydb-v1-java" local res=0 cd "${script_dir}/resources" - $("${script_dir}"/../generate_library.sh \ + bash "${script_dir}"/../generate_library.sh \ -p google/cloud/alloydb/v1 \ -d ../"${destination}" \ --gapic_generator_version 2.24.0 \ --protobuf_version 22.99 \ --grpc_version 1.55.1 \ --transport grpc+rest \ - --rest_numeric_enums true \ - --os_architecture "$(__get_os_architecture)") || res=$? + --rest_numeric_enums true || res=$? assertEquals 1 $((res)) # still need to clean up potential downloaded tooling. cleanup "${destination}" @@ -217,14 +215,13 @@ generate_library_failed_with_invalid_grpc_version() { local destination="google-cloud-alloydb-v1-java" local res=0 cd "${script_dir}/resources" - $("${script_dir}"/../generate_library.sh \ + bash "${script_dir}"/../generate_library.sh \ -p google/cloud/alloydb/v1 \ - -d ../"${destination}" \ + -d ../output/"${destination}" \ --gapic_generator_version 2.24.0 \ --grpc_version 0.99.0 \ --transport grpc+rest \ - --rest_numeric_enums true \ - --os_architecture "$(__get_os_architecture)") || res=$? + --rest_numeric_enums true || res=$? assertEquals 1 $((res)) # still need to clean up potential downloaded tooling. cleanup "${destination}" diff --git a/library_generation/utilities.sh b/library_generation/utilities.sh index 804d964fc9..39acc0e89b 100755 --- a/library_generation/utilities.sh +++ b/library_generation/utilities.sh @@ -2,6 +2,7 @@ set -xeo pipefail + # private functions that should not be called outside this file. # Used to obtain configuration values from a bazel BUILD file @@ -157,6 +158,7 @@ get_protobuf_version() { } download_tools() { + pushd "${output_folder}" local gapic_generator_version=$1 local protobuf_version=$2 local grpc_version=$3 @@ -164,6 +166,7 @@ download_tools() { download_generator "${gapic_generator_version}" download_protobuf "${protobuf_version}" "${os_architecture}" download_grpc_plugin "${grpc_version}" "${os_architecture}" + popd } download_generator() { @@ -197,7 +200,7 @@ download_protobuf() { rm "protobuf-${protobuf_version}.zip" fi - protoc_path=protobuf-${protobuf_version}/bin + protoc_path="${output_folder}/protobuf-${protobuf_version}/bin" } download_grpc_plugin() { @@ -305,13 +308,13 @@ sparse_clone() { clone_dir=$(basename "${repo_url%.*}") rm -rf "${clone_dir}" git clone -n --depth=1 --no-single-branch --filter=tree:0 "${repo_url}" - cd "${clone_dir}" + pushd "${clone_dir}" if [ -n "${commitish}" ]; then git checkout "${commitish}" fi git sparse-checkout set --no-cone ${paths} git checkout - cd .. + popd } # takes a versions.txt file and returns its version @@ -322,6 +325,13 @@ get_version_from_versions_txt() { echo "${version}" } +# gets the output folder where all sources and dependencies will be located. It +# relies on utilities_script_dir which points to the same location as +# `generate_library.sh` +get_output_folder() { + echo "$(pwd)/output" +} + detect_os_architecture() { local os_type local os_architecture diff --git a/showcase/scripts/generate_showcase.sh b/showcase/scripts/generate_showcase.sh index 4d25ca3155..6619f5801d 100755 --- a/showcase/scripts/generate_showcase.sh +++ b/showcase/scripts/generate_showcase.sh @@ -11,6 +11,7 @@ source "${lib_gen_scripts_dir}/utilities.sh" readonly perform_cleanup=$1 cd "${SCRIPT_DIR}" +mkdir -p "${SCRIPT_DIR}/output" # clone gapic-showcase if [ ! -d schema ]; then @@ -23,7 +24,7 @@ if [ ! -d schema ]; then showcase_version=$(grep -e '' "${SCRIPT_DIR}/../gapic-showcase/pom.xml" | cut -d'>' -f2 | cut -d'<' -f1) sparse_clone https://github.com/googleapis/gapic-showcase.git "schema/google/showcase/v1beta1" "v${showcase_version}" cd gapic-showcase - mv schema .. + mv schema ../output cd .. rm -rdf gapic-showcase fi @@ -32,7 +33,7 @@ if [ ! -d google ];then rm -rdf googleapis fi sparse_clone https://github.com/googleapis/googleapis.git "WORKSPACE google/api google/rpc google/cloud/common_resources.proto google/longrunning google/iam/v1 google/cloud/location google/type" - mv googleapis/google . + mv googleapis/google output rm -rdf googleapis fi @@ -41,8 +42,8 @@ ggj_version=$(get_version_from_versions_txt ../../versions.txt "gapic-generator- rest_numeric_enums="false" transport="grpc+rest" include_samples="false" -rm -rdf showcase-output -mkdir showcase-output +rm -rdf output/showcase-output +mkdir output/showcase-output set +e bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ --proto_path "schema/google/showcase/v1beta1" \ @@ -50,11 +51,11 @@ bash "${SCRIPT_DIR}/../../library_generation/generate_library.sh" \ --gapic_generator_version "${ggj_version}" \ --rest_numeric_enums "${rest_numeric_enums}" \ --include_samples "${include_samples}" \ - --transport "${transport}" &> out + --transport "${transport}" exit_code=$? if [ "${exit_code}" -ne 0 ]; then - rm -rdf showcase-output + rm -rdf output exit "${exit_code}" fi set +x diff --git a/showcase/scripts/showcase_utilities.sh b/showcase/scripts/showcase_utilities.sh index 8db80248ca..591486f753 100644 --- a/showcase/scripts/showcase_utilities.sh +++ b/showcase/scripts/showcase_utilities.sh @@ -3,5 +3,5 @@ function cleanup { script_dir=$1 cd "${script_dir}" - rm -rdf gapic-generator-java* google schema protobuf-* protoc-gen-grpc-java* showcase-output out + rm -rdf output gapic-generator-java* } diff --git a/showcase/scripts/verify.sh b/showcase/scripts/verify.sh index e3d4c61392..b45e18ecb7 100755 --- a/showcase/scripts/verify.sh +++ b/showcase/scripts/verify.sh @@ -12,16 +12,26 @@ readonly REPLACE_SOURCE='false' readonly PERFORM_CLEANUP='false' bash "${SCRIPT_DIR}/generate_components.sh" "${REPLACE_SOURCE}" "${PERFORM_CLEANUP}" -# compare with proto library PROTO_PROJECT_DIR='proto-gapic-showcase-v1beta1' -diff -ru "${SHOWCASE_DIR}/${PROTO_PROJECT_DIR}/src" "${SCRIPT_DIR}/showcase-output/proto-showcase-output/src" - -# compare with grpc library GRPC_PROJECT_DIR='grpc-gapic-showcase-v1beta1' -diff -ru "${SHOWCASE_DIR}/${GRPC_PROJECT_DIR}/src/main/java/com" "${SCRIPT_DIR}/showcase-output/grpc-showcase-output/src/main/java/com" - -# compare with gapic library GAPIC_PROJECT_DIR='gapic-showcase' -diff -ru "${SHOWCASE_DIR}/${GAPIC_PROJECT_DIR}/src/main/java" "${SCRIPT_DIR}/showcase-output/gapic-showcase-output/src/main/java" + +{ + # compare with proto library + (diff -ru "${SHOWCASE_DIR}/${PROTO_PROJECT_DIR}/src" "${SCRIPT_DIR}/output/showcase-output/proto-showcase-output/src") && + + # compare with grpc library + (diff -ru "${SHOWCASE_DIR}/${GRPC_PROJECT_DIR}/src/main/java/com" "${SCRIPT_DIR}/output/showcase-output/grpc-showcase-output/src/main/java/com") && + + # compare with gapic library + (diff -ru "${SHOWCASE_DIR}/${GAPIC_PROJECT_DIR}/src/main/java" "${SCRIPT_DIR}/output/showcase-output/gapic-showcase-output/src/main/java") +} || { + failure="true" +} cleanup $SCRIPT_DIR + +if [ "${failure}" == "true" ]; then + exit 1 +fi +