diff --git a/build/cloud_spanner_emulator/defs.bzl b/build/cloud_spanner_emulator/defs.bzl index c1c092a03..81dc1e420 100644 --- a/build/cloud_spanner_emulator/defs.bzl +++ b/build/cloud_spanner_emulator/defs.bzl @@ -41,6 +41,6 @@ _cloud_spanner_emulator_binaries = repository_rule( def cloud_spanner_emulator_binaries(): _cloud_spanner_emulator_binaries( name = "cloud_spanner_emulator", - sha256 = "7a3cdd5db7f5a427230ab67a8dc09cfcb6752dd7f0b28d51e8d08150b2641506", - version = "1.1.1", + sha256 = "7048efb8708e0b2af98d84c9bd86ecb158b5663116c5dea789974d6a616f0e12", + version = "1.4.2", ) diff --git a/build/com_github_grpc_grpc/repo.bzl b/build/com_github_grpc_grpc/repo.bzl index 10f7e84e8..b00cf045e 100644 --- a/build/com_github_grpc_grpc/repo.bzl +++ b/build/com_github_grpc_grpc/repo.bzl @@ -22,7 +22,14 @@ def com_github_grpc_grpc_repo(): if "com_github_grpc_grpc" not in native.existing_rules(): http_archive( name = "com_github_grpc_grpc", - sha256 = "8eb9d86649c4d4a7df790226df28f081b97a62bf12c5c5fe9b5d31a29cd6541a", - strip_prefix = "grpc-1.36.4", - urls = ["https://github.com/grpc/grpc/archive/v1.36.4.tar.gz"], + sha256 = "67423a4cd706ce16a88d1549297023f0f9f0d695a96dd684adc21e67b021f9bc", + strip_prefix = "grpc-1.46.0", + urls = ["https://github.com/grpc/grpc/archive/v1.46.0.tar.gz"], ) + +# http_archive( +# name = "com_github_grpc_grpc", +# sha256 = "46ee3d12c6d42a4ded46ba025c8ba09850425312c710c6ee6e084bd808ebc1e8", +# strip_prefix = "grpc-1.45.1", +# urls = ["https://github.com/grpc/grpc/archive/v1.45.1.tar.gz"], +# ) diff --git a/build/com_github_grpc_grpc_kotlin/repo.bzl b/build/com_github_grpc_grpc_kotlin/repo.bzl index 3e5519358..503503de5 100644 --- a/build/com_github_grpc_grpc_kotlin/repo.bzl +++ b/build/com_github_grpc_grpc_kotlin/repo.bzl @@ -29,8 +29,14 @@ def com_github_grpc_grpc_kotlin_repo(): if "io_grpc_grpc_java" not in native.existing_rules(): http_archive( name = "io_grpc_grpc_java", - sha256 = "85927f857e0b3ad5c4e51c2e6d29213d3e0319f20784aa2113552f71311ba74c", - # Latest compatible version per https://github.com/grpc/grpc-kotlin/blob/v1.2.1/compiler/README.md - strip_prefix = "grpc-java-1.39.0", - url = "https://github.com/grpc/grpc-java/archive/refs/tags/v1.39.0.tar.gz", + sha256 = "c1b80883511ceb1e433fb2d4b2f6d85dca0c62a265a6a3e6695144610d6f65b8", + strip_prefix = "grpc-java-1.46.0", + url = "https://github.com/grpc/grpc-java/archive/refs/tags/v1.46.0.tar.gz", ) + +# http_archive( +# name = "io_grpc_grpc_java", +# sha256 = "51bac553d269b97214dbd6aee4e65fc616d8ccd414fc12d708e85979ed4c19b4", +# strip_prefix = "grpc-java-1.45.1", +# url = "https://github.com/grpc/grpc-java/archive/refs/tags/v1.45.1.tar.gz", +# ) diff --git a/build/common_jvm_deps.bzl b/build/common_jvm_deps.bzl index 2a83e6c4c..93191c964 100644 --- a/build/common_jvm_deps.bzl +++ b/build/common_jvm_deps.bzl @@ -24,20 +24,18 @@ load("//build/io_bazel_rules_kotlin:deps.bzl", "rules_kotlin_deps") load("//build/io_bazel_rules_docker:base_images.bzl", "base_java_images") load("@com_github_grpc_grpc//bazel:grpc_deps.bzl", "grpc_deps") load("@com_google_googleapis//:repository_rules.bzl", "switched_rules_by_language") -load("//build/io_bazel_rules_go:deps.bzl", "rules_go_deps") load("@tink_java//:tink_java_deps.bzl", "tink_java_deps") def common_jvm_deps(): """ Adds all external repos necessary for common-jvm. """ + grpc_deps() container_repositories() rules_kotlin_deps() base_java_images() - grpc_deps() switched_rules_by_language( name = "com_google_googleapis_imports", java = True, ) - rules_go_deps() tink_java_deps() diff --git a/build/common_jvm_extra_deps.bzl b/build/common_jvm_extra_deps.bzl index d47b4560e..ce015052d 100644 --- a/build/common_jvm_extra_deps.bzl +++ b/build/common_jvm_extra_deps.bzl @@ -31,8 +31,8 @@ def common_jvm_extra_deps(): Adds all external repos necessary for common-jvm. """ compat_repositories() + grpc_extra_deps() container_deps() java_image_repositories() - grpc_extra_deps() grpc_kt_repositories() grpc_java_repositories() diff --git a/build/common_jvm_maven.bzl b/build/common_jvm_maven.bzl index 325fe7867..7d1d3afbd 100644 --- a/build/common_jvm_maven.bzl +++ b/build/common_jvm_maven.bzl @@ -58,11 +58,9 @@ def common_jvm_maven_artifacts(): # or default dependency versions). maven_artifacts.update({ "com.adobe.testing:s3mock-junit4": "2.2.3", - "com.google.api:gax-grpc": "2.12.2", - "com.google.api.grpc:grpc-core-proto": "0.0.6", - "com.google.cloud:google-cloud-bigquery": "2.4.1", - "com.google.cloud:google-cloud-nio": "0.123.10", - "com.google.cloud:google-cloud-spanner": "3.0.3", + "com.google.cloud:google-cloud-bigquery": "2.10.10", + "com.google.cloud:google-cloud-nio": "0.123.28", + "com.google.cloud:google-cloud-spanner": "6.23.3", "com.google.guava:guava": "31.0.1-jre", "info.picocli:picocli": "4.4.0", "junit:junit": "4.13", @@ -75,6 +73,7 @@ def common_jvm_maven_artifacts(): # Liquibase. "org.liquibase:liquibase-core": "4.9.1", "com.google.cloudspannerecosystem:liquibase-spanner": "4.6.1", + "com.google.cloud:google-cloud-spanner-jdbc": "2.6.4", # For grpc-kotlin. This should be a version that is compatible with the # Kotlin release used by rules_kotlin. diff --git a/build/common_jvm_repositories.bzl b/build/common_jvm_repositories.bzl index e893c4095..27f333918 100644 --- a/build/common_jvm_repositories.bzl +++ b/build/common_jvm_repositories.bzl @@ -30,7 +30,6 @@ load( "//build/io_bazel_rules_kotlin:repo.bzl", "rules_kotlin_repo", ) -load("//build/io_bazel_rules_go:repo.bzl", "rules_go_repo") load("//build/tink:repo.bzl", "tink_java") def common_jvm_repositories(): @@ -48,5 +47,4 @@ def common_jvm_repositories(): grpc_health_probe_repo() rules_docker_repo() com_google_googleapis_repo() - rules_go_repo() tink_java() diff --git a/build/io_bazel_rules_go/BUILD.bazel b/build/io_bazel_rules_go/BUILD.bazel deleted file mode 100644 index e69de29bb..000000000 diff --git a/build/io_bazel_rules_go/deps.bzl b/build/io_bazel_rules_go/deps.bzl deleted file mode 100644 index 78f1d9d5c..000000000 --- a/build/io_bazel_rules_go/deps.bzl +++ /dev/null @@ -1,26 +0,0 @@ -# Copyright 2021 The Cross-Media Measurement Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Dependencies of rules_go.""" - -load("@io_bazel_rules_go//go:deps.bzl", "go_register_toolchains", "go_rules_dependencies") -load("@bazel_gazelle//:deps.bzl", "gazelle_dependencies") - -_GO_VERSION = "1.17.3" - -def rules_go_deps(): - go_rules_dependencies() - go_register_toolchains(version = _GO_VERSION) - - gazelle_dependencies() diff --git a/build/io_bazel_rules_go/repo.bzl b/build/io_bazel_rules_go/repo.bzl deleted file mode 100644 index 69cd92a27..000000000 --- a/build/io_bazel_rules_go/repo.bzl +++ /dev/null @@ -1,41 +0,0 @@ -# Copyright 2021 The Cross-Media Measurement Authors -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -"""Repository targets for rules_go.""" - -load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") - -_REPO_NAME = "io_bazel_rules_go" -_VERSION = "0.29.0" -_GAZELLE_VERSION = "0.24.0" - -def rules_go_repo(): - if _REPO_NAME not in native.existing_rules(): - http_archive( - name = _REPO_NAME, - sha256 = "2b1641428dff9018f9e85c0384f03ec6c10660d935b750e3fa1492a281a53b0f", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/rules_go/releases/download/v{version}/rules_go-v{version}.zip".format(version = _VERSION), - "https://github.com/bazelbuild/rules_go/releases/download/v{version}/rules_go-v{version}.zip".format(version = _VERSION), - ], - ) - - http_archive( - name = "bazel_gazelle", - sha256 = "de69a09dc70417580aabf20a28619bb3ef60d038470c7cf8442fafcf627c21cb", - urls = [ - "https://mirror.bazel.build/github.com/bazelbuild/bazel-gazelle/releases/download/v{version}/bazel-gazelle-v{version}.tar.gz".format(version = _GAZELLE_VERSION), - "https://github.com/bazelbuild/bazel-gazelle/releases/download/v{version}/bazel-gazelle-v{version}.tar.gz".format(version = _GAZELLE_VERSION), - ], - ) diff --git a/imports/java/com/google/api/grpc/BUILD.bazel b/imports/java/com/google/api/grpc/BUILD.bazel deleted file mode 100644 index 1a67993aa..000000000 --- a/imports/java/com/google/api/grpc/BUILD.bazel +++ /dev/null @@ -1,6 +0,0 @@ -package(default_visibility = ["//visibility:public"]) - -alias( - name = "grpc", - actual = "@maven//:com_google_api_grpc_grpc_core_proto", -) diff --git a/src/main/kotlin/org/wfanet/measurement/gcloud/common/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/gcloud/common/BUILD.bazel index 93bba3b7d..28b47f181 100644 --- a/src/main/kotlin/org/wfanet/measurement/gcloud/common/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/gcloud/common/BUILD.bazel @@ -9,9 +9,9 @@ kt_jvm_library( srcs = glob(["*.kt"]), deps = [ "//imports/java/com/google/api/core", - "//imports/java/com/google/api/grpc", "//imports/java/com/google/cloud:core", "//imports/java/com/google/protobuf", "//imports/kotlin/kotlinx/coroutines:core", + "@com_google_googleapis//google/type:type_java_proto", ], ) diff --git a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/BUILD.bazel index 3bc4e4855..7d76382f4 100644 --- a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/BUILD.bazel +++ b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/BUILD.bazel @@ -4,12 +4,23 @@ load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") package(default_visibility = ["//visibility:public"]) +kt_jvm_library( + name = "flags", + srcs = ["SpannerFlags.kt"], + deps = ["//imports/java/picocli"], +) + kt_jvm_library( name = "spanner", - srcs = glob(["*.kt"]), + srcs = glob( + ["*.kt"], + exclude = ["SpannerFlags.kt"], + ), + resources = ["//src/main/resources/META-INF/services:grpclb"], + exports = [":flags"], deps = [ + ":flags", "//imports/java/com/google/cloud/spanner", - "//imports/java/picocli", "//imports/kotlin/kotlinx/coroutines:core", "//src/main/kotlin/org/wfanet/measurement/common", "//src/main/kotlin/org/wfanet/measurement/common/identity", diff --git a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerDatabaseConnector.kt b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerDatabaseConnector.kt index 781baaca1..67888243b 100644 --- a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerDatabaseConnector.kt +++ b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerDatabaseConnector.kt @@ -17,6 +17,8 @@ package org.wfanet.measurement.gcloud.spanner import com.google.cloud.spanner.DatabaseClient import com.google.cloud.spanner.DatabaseId import com.google.cloud.spanner.Spanner +import io.grpc.LoadBalancerRegistry +import io.grpc.grpclb.GrpclbLoadBalancerProvider import java.time.Duration import java.util.logging.Logger import kotlinx.coroutines.TimeoutCancellationException @@ -78,3 +80,24 @@ class SpannerDatabaseConnector( private val logger = Logger.getLogger(this::class.java.name) } } + +/** Builds a [SpannerDatabaseConnector] from these flags. */ +private fun SpannerFlags.toSpannerDatabaseConnector(): SpannerDatabaseConnector { + return SpannerDatabaseConnector( + projectName = projectName, + instanceName = instanceName, + databaseName = databaseName, + readyTimeout = readyTimeout, + emulatorHost = emulatorHost + ) +} + +/** + * Executes [block] with a [SpannerDatabaseConnector] resource once it's ready, ensuring that the + * resource is closed. + */ +suspend fun SpannerFlags.usingSpanner( + block: suspend (spanner: SpannerDatabaseConnector) -> R +): R { + return toSpannerDatabaseConnector().usingSpanner(block) +} diff --git a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerFlags.kt b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerFlags.kt index fcc3b09fd..5322f1cde 100644 --- a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerFlags.kt +++ b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/SpannerFlags.kt @@ -56,25 +56,6 @@ class SpannerFlags { description = ["Host name and port of the spanner emulator."], required = false ) - var spannerEmulatorHost: String? = null + var emulatorHost: String? = null private set - - /** Builds a [SpannerDatabaseConnector] from these flags. */ - private fun toSpannerDatabaseConnector(): SpannerDatabaseConnector { - return SpannerDatabaseConnector( - projectName = projectName, - instanceName = instanceName, - databaseName = databaseName, - readyTimeout = readyTimeout, - emulatorHost = spannerEmulatorHost - ) - } - - /** - * Executes [block] with a [SpannerDatabaseConnector] resource once it's ready, ensuring that the - * resource is closed. - */ - suspend fun usingSpanner(block: suspend (spanner: SpannerDatabaseConnector) -> R): R { - return toSpannerDatabaseConnector().usingSpanner(block) - } } diff --git a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools/BUILD.bazel b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools/BUILD.bazel new file mode 100644 index 000000000..96af52b56 --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools/BUILD.bazel @@ -0,0 +1,15 @@ +load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library") + +package(default_visibility = ["//visibility:public"]) + +kt_jvm_library( + name = "update_schema", + srcs = ["UpdateSchema.kt"], + resources = ["//src/main/resources/META-INF/services:grpclb"], + runtime_deps = ["//imports/java/liquibase/ext/spanner"], + deps = [ + "//src/main/kotlin/org/wfanet/measurement/common", + "//src/main/kotlin/org/wfanet/measurement/common/db/liquibase", + "//src/main/kotlin/org/wfanet/measurement/gcloud/spanner:flags", + ], +) diff --git a/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools/UpdateSchema.kt b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools/UpdateSchema.kt new file mode 100644 index 000000000..457788e2c --- /dev/null +++ b/src/main/kotlin/org/wfanet/measurement/gcloud/spanner/tools/UpdateSchema.kt @@ -0,0 +1,76 @@ +/* + * Copyright 2022 The Cross-Media Measurement Authors + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.wfanet.measurement.gcloud.spanner.tools + +import java.sql.DriverManager +import java.util.logging.Level +import java.util.logging.Logger +import liquibase.Contexts +import liquibase.Scope +import org.wfanet.measurement.common.commandLineMain +import org.wfanet.measurement.common.db.liquibase.Liquibase +import org.wfanet.measurement.common.db.liquibase.setLogLevel +import org.wfanet.measurement.common.getJarResourcePath +import org.wfanet.measurement.gcloud.spanner.SpannerFlags +import picocli.CommandLine.Command +import picocli.CommandLine.Mixin +import picocli.CommandLine.Option + +@Command +class UpdateSchema : Runnable { + @Mixin private lateinit var flags: SpannerFlags + + @Option( + names = ["--changelog"], + description = ["Liquibase changelog resource name"], + required = true, + ) + private lateinit var changelog: String + + override fun run() { + val connectionString = + if (flags.emulatorHost == null) { + "jdbc:cloudspanner:/projects/${flags.projectName}/instances/${flags.instanceName}/" + + "databases/${flags.databaseName}" + } else { + "jdbc:cloudspanner://${flags.emulatorHost}/projects/${flags.projectName}/instances/" + + "${flags.instanceName}/databases/${flags.databaseName}" + + ";usePlainText=true;autoConfigEmulator=true" + } + + val changelogPath = + checkNotNull(Thread.currentThread().contextClassLoader.getJarResourcePath(changelog)) { + "JAR resource $changelog not found" + } + + logger.info("Connecting to $connectionString") + DriverManager.getConnection(connectionString).use { connection -> + logger.info("Loading changelog from $changelogPath") + Liquibase.fromPath(connection, changelogPath).use { liquibase -> + logger.info("Updating...") + Scope.getCurrentScope().setLogLevel(Level.FINE) + liquibase.update(Contexts()) + } + } + } + + companion object { + private val logger = Logger.getLogger(this::class.java.name) + + @JvmStatic fun main(args: Array) = commandLineMain(UpdateSchema(), args) + } +} diff --git a/src/main/resources/META-INF/services/BUILD.bazel b/src/main/resources/META-INF/services/BUILD.bazel new file mode 100644 index 000000000..8417c7246 --- /dev/null +++ b/src/main/resources/META-INF/services/BUILD.bazel @@ -0,0 +1,8 @@ +package(default_visibility = ["//visibility:public"]) + +# TODO(grpc/grpc-java#9149): Remove once @io_grpc_grpc_java//grpclb contains +# these resources. +filegroup( + name = "grpclb", + srcs = glob(["io.grpc.*"]), +) diff --git a/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider b/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider new file mode 100644 index 000000000..581711343 --- /dev/null +++ b/src/main/resources/META-INF/services/io.grpc.LoadBalancerProvider @@ -0,0 +1 @@ +io.grpc.grpclb.GrpclbLoadBalancerProvider diff --git a/src/main/resources/META-INF/services/io.grpc.NameResolverProvider b/src/main/resources/META-INF/services/io.grpc.NameResolverProvider new file mode 100644 index 000000000..bf02b5e84 --- /dev/null +++ b/src/main/resources/META-INF/services/io.grpc.NameResolverProvider @@ -0,0 +1 @@ +io.grpc.grpclb.SecretGrpclbNameResolverProvider$Provider