From b3bb0f9847497bc97951172d248fad3c88096829 Mon Sep 17 00:00:00 2001 From: Kuat Date: Tue, 15 Dec 2020 16:37:00 -0800 Subject: [PATCH] deps: update protobuf to 3.14 (#14253) There is an unfortunate change in 3.14 that changed go_package for WKT, which necessitates updating several go dependencies as well: protoc-gen-validate to 2020-11-30 transitive dependencies from protoc-gen-validate gazelle to 0.22.2 rules_go to 0.25.0 go to 1.15.5 Risk Level: low Signed-off-by: Kuat Yessenov Co-authored-by: Lizan Zhou --- .azure-pipelines/bazel.yml | 2 ++ api/bazel/api_build_system.bzl | 16 +++++++-------- api/bazel/repository_locations.bzl | 19 ++++++++++++------ api/test/build/BUILD | 2 -- api/test/build/go_build_test.go | 2 -- bazel/dependency_imports.bzl | 20 ++++++++++++++++++- bazel/protobuf.patch | 14 +++++++++++++ bazel/repository_locations.bzl | 18 ++++++++--------- .../bazel/api_build_system.bzl | 16 +++++++-------- .../bazel/repository_locations.bzl | 19 ++++++++++++------ source/common/config/protobuf_link_hacks.h | 1 + .../common/access_log/access_log_impl_test.cc | 2 +- .../upstream/cluster_manager_impl_test.cc | 2 +- test/common/upstream/eds_speed_test.cc | 1 + test/common/upstream/upstream_impl_test.cc | 7 ++----- .../filters/http/compressor/config_test.cc | 3 +++ test/integration/BUILD | 1 + test/integration/integration_test.cc | 3 +++ 18 files changed, 99 insertions(+), 49 deletions(-) diff --git a/.azure-pipelines/bazel.yml b/.azure-pipelines/bazel.yml index 62cb412e588f..3c440cdef934 100644 --- a/.azure-pipelines/bazel.yml +++ b/.azure-pipelines/bazel.yml @@ -66,6 +66,8 @@ steps: - bash: | echo "disk space at end of build:" df -h + # Cleanup offending files with unicode names + rm -rf $(Build.StagingDirectory)/tmp/*/*/external/go_sdk/test/fixedbugs displayName: "Check disk space at end" condition: always() diff --git a/api/bazel/api_build_system.bzl b/api/bazel/api_build_system.bzl index c0269d161f80..8a0e0bf71021 100644 --- a/api/bazel/api_build_system.bzl +++ b/api/bazel/api_build_system.bzl @@ -186,14 +186,14 @@ def api_proto_package( proto = name, visibility = ["//visibility:public"], deps = depset([_go_proto_mapping(dep) for dep in deps] + [ - "@com_github_golang_protobuf//ptypes:go_default_library", - "@com_github_golang_protobuf//ptypes/any:go_default_library", - "@com_github_golang_protobuf//ptypes/duration:go_default_library", - "@com_github_golang_protobuf//ptypes/struct:go_default_library", - "@com_github_golang_protobuf//ptypes/timestamp:go_default_library", - "@com_github_golang_protobuf//ptypes/wrappers:go_default_library", "@com_envoyproxy_protoc_gen_validate//validate:go_default_library", - "@com_google_googleapis//google/api:annotations_go_proto", - "@com_google_googleapis//google/rpc:status_go_proto", + "@com_github_golang_protobuf//ptypes:go_default_library_gen", + "@go_googleapis//google/api:annotations_go_proto", + "@go_googleapis//google/rpc:status_go_proto", + "@io_bazel_rules_go//proto/wkt:any_go_proto", + "@io_bazel_rules_go//proto/wkt:duration_go_proto", + "@io_bazel_rules_go//proto/wkt:struct_go_proto", + "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", + "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", ]).to_list(), ) diff --git a/api/bazel/repository_locations.bzl b/api/bazel/repository_locations.bzl index f0173e58946e..68ea3e2ffb84 100644 --- a/api/bazel/repository_locations.bzl +++ b/api/bazel/repository_locations.bzl @@ -14,23 +14,30 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "protoc-gen-validate (PGV)", project_desc = "protoc plugin to generate polyglot message validators", project_url = "https://github.com/envoyproxy/protoc-gen-validate", - version = "278964a8052f96a2f514add0298098f63fb7f47f", - sha256 = "e368733c9fb7f8489591ffaf269170d7658cc0cd1ee322b601512b769446d3c8", + version = "1bcea29601b5624234a19b3d7f0ebd9e9984f583", + sha256 = "2062bbe50eddf3c98490339721fb02b5b5cd78f610f163b98bbf95ba7105553f", strip_prefix = "protoc-gen-validate-{version}", urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/{version}.tar.gz"], - release_date = "2020-06-08", + release_date = "2020-11-30", use_category = ["api"], + implied_untracked_deps = [ + "com_github_iancoleman_strcase", + "com_github_lyft_protoc_gen_star", + "com_github_spf13_afero", + "org_golang_google_genproto", + "org_golang_x_text", + ], ), com_github_cncf_udpa = dict( project_name = "xDS API", project_desc = "xDS API Working Group (xDS-WG)", project_url = "https://github.com/cncf/udpa", # During the UDPA -> xDS migration, we aren't working with releases. - version = "5459f2c994033b0afed7e4a70ac7e90c90c1ffee", - sha256 = "c1f5c2438cf725b5f66aa4210dbc4bb691020c5ed4f64d2bc6638b06a11482f1", + version = "cc1b757b3eddccaaaf0743cbb107742bb7e3ee4f", + sha256 = "822a007cf155855d0c08a2e753a39e222e5816b904436196244066a818a8a230", strip_prefix = "udpa-{version}", urls = ["https://github.com/cncf/udpa/archive/{version}.tar.gz"], - release_date = "2020-11-20", + release_date = "2020-12-11", use_category = ["api"], ), com_github_openzipkin_zipkinapi = dict( diff --git a/api/test/build/BUILD b/api/test/build/BUILD index 2dae9fa0de03..affcc0403496 100644 --- a/api/test/build/BUILD +++ b/api/test/build/BUILD @@ -23,11 +23,9 @@ api_go_test( deps = [ "//envoy/api/v2:pkg_go_proto", "//envoy/api/v2/auth:pkg_go_proto", - "//envoy/config/bootstrap/v2:pkg_go_proto", "//envoy/service/accesslog/v2:pkg_go_proto", "//envoy/service/discovery/v2:pkg_go_proto", "//envoy/service/metrics/v2:pkg_go_proto", "//envoy/service/ratelimit/v2:pkg_go_proto", - "//envoy/service/trace/v2:pkg_go_proto", ], ) diff --git a/api/test/build/go_build_test.go b/api/test/build/go_build_test.go index 638ef478b8c7..e350404b3ab6 100644 --- a/api/test/build/go_build_test.go +++ b/api/test/build/go_build_test.go @@ -5,12 +5,10 @@ import ( _ "github.com/envoyproxy/go-control-plane/envoy/api/v2" _ "github.com/envoyproxy/go-control-plane/envoy/api/v2/auth" - _ "github.com/envoyproxy/go-control-plane/envoy/config/bootstrap/v2" _ "github.com/envoyproxy/go-control-plane/envoy/service/accesslog/v2" _ "github.com/envoyproxy/go-control-plane/envoy/service/discovery/v2" _ "github.com/envoyproxy/go-control-plane/envoy/service/metrics/v2" _ "github.com/envoyproxy/go-control-plane/envoy/service/ratelimit/v2" - _ "github.com/envoyproxy/go-control-plane/envoy/service/trace/v2" ) func TestNoop(t *testing.T) { diff --git a/bazel/dependency_imports.bzl b/bazel/dependency_imports.bzl index 62b71950c9ed..f2ef292a2d93 100644 --- a/bazel/dependency_imports.bzl +++ b/bazel/dependency_imports.bzl @@ -16,7 +16,7 @@ load("@rules_antlr//antlr:deps.bzl", "antlr_dependencies") load("@proxy_wasm_rust_sdk//bazel:dependencies.bzl", "proxy_wasm_rust_sdk_dependencies") # go version for rules_go -GO_VERSION = "1.14.7" +GO_VERSION = "1.15.5" def envoy_dependency_imports(go_version = GO_VERSION): rules_foreign_cc_dependencies() @@ -57,6 +57,24 @@ def envoy_dependency_imports(go_version = GO_VERSION): sum = "h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=", version = "v0.3.0", ) + go_repository( + name = "com_github_spf13_afero", + importpath = "github.com/spf13/afero", + sum = "h1:8q6vk3hthlpb2SouZcnBVKboxWQWMDNF38bwholZrJc=", + version = "v1.3.4", + ) + go_repository( + name = "com_github_lyft_protoc_gen_star", + importpath = "github.com/lyft/protoc-gen-star", + sum = "h1:sImehRT+p7lW9n6R7MQc5hVgzWGEkDVZU4AsBQ4Isu8=", + version = "v0.5.1", + ) + go_repository( + name = "com_github_iancoleman_strcase", + importpath = "github.com/iancoleman/strcase", + sum = "h1:ux/56T2xqZO/3cP1I2F86qpeoYPCOzk+KF/UH/Ar+lk=", + version = "v0.0.0-20180726023541-3605ed457bf7", + ) config_validation_pip_install() configs_pip_install() diff --git a/bazel/protobuf.patch b/bazel/protobuf.patch index 1e19ebbb7ffe..fd6325838ba4 100644 --- a/bazel/protobuf.patch +++ b/bazel/protobuf.patch @@ -21,3 +21,17 @@ index efc3d8e7f..746ad4851 100644 ################################################################################ # Protobuf Runtime Library +diff --git a/python/google/protobuf/__init__.py b/python/google/protobuf/__init__.py +index 97ac28028..8b7585d9d 100644 +--- a/python/google/protobuf/__init__.py ++++ b/python/google/protobuf/__init__.py +@@ -31,3 +31,9 @@ + # Copyright 2007 Google Inc. All Rights Reserved. + + __version__ = '3.14.0' ++ ++if __name__ != '__main__': ++ try: ++ __import__('pkg_resources').declare_namespace(__name__) ++ except ImportError: ++ __path__ = __import__('pkgutil').extend_path(__path__, __name__) diff --git a/bazel/repository_locations.bzl b/bazel/repository_locations.bzl index 747d36d610db..458c69f1f82b 100644 --- a/bazel/repository_locations.bzl +++ b/bazel/repository_locations.bzl @@ -15,10 +15,10 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Gazelle", project_desc = "Bazel BUILD file generator for Go projects", project_url = "https://github.com/bazelbuild/bazel-gazelle", - version = "0.21.1", - sha256 = "cdb02a887a7187ea4d5a27452311a75ed8637379a1287d8eeb952138ea485f7d", + version = "0.22.2", + sha256 = "b85f48fa105c4403326e9525ad2b2cc437babaa6e15a3fc0b1dbab0ab064bc7c", urls = ["https://github.com/bazelbuild/bazel-gazelle/releases/download/v{version}/bazel-gazelle-v{version}.tar.gz"], - release_date = "2020-05-28", + release_date = "2020-10-02", use_category = ["build"], ), bazel_toolchains = dict( @@ -495,12 +495,12 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Protocol Buffers", project_desc = "Language-neutral, platform-neutral extensible mechanism for serializing structured data", project_url = "https://developers.google.com/protocol-buffers", - version = "3.13.0", - sha256 = "465fd9367992a9b9c4fba34a549773735da200903678b81b25f367982e8df376", + version = "3.14.0", + sha256 = "6dd0f6b20094910fbb7f1f7908688df01af2d4f6c5c21331b9f636048674aebf", strip_prefix = "protobuf-{version}", urls = ["https://github.com/protocolbuffers/protobuf/releases/download/v{version}/protobuf-all-{version}.tar.gz"], use_category = ["dataplane_core", "controlplane"], - release_date = "2020-08-14", + release_date = "2020-11-13", cpe = "cpe:2.3:a:google:protobuf:*", ), grpc_httpjson_transcoding = dict( @@ -520,11 +520,11 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "Go rules for Bazel", project_desc = "Bazel rules for the Go language", project_url = "https://github.com/bazelbuild/rules_go", - version = "0.23.7", - sha256 = "0310e837aed522875791750de44408ec91046c630374990edd51827cb169f616", + version = "0.25.0", + sha256 = "6f111c57fd50baf5b8ee9d63024874dd2a014b069426156c55adbf6d3d22cb7b", urls = ["https://github.com/bazelbuild/rules_go/releases/download/v{version}/rules_go-v{version}.tar.gz"], use_category = ["build", "api"], - release_date = "2020-08-06", + release_date = "2020-12-02", implied_untracked_deps = [ "com_github_golang_protobuf", "io_bazel_rules_nogo", diff --git a/generated_api_shadow/bazel/api_build_system.bzl b/generated_api_shadow/bazel/api_build_system.bzl index c0269d161f80..8a0e0bf71021 100644 --- a/generated_api_shadow/bazel/api_build_system.bzl +++ b/generated_api_shadow/bazel/api_build_system.bzl @@ -186,14 +186,14 @@ def api_proto_package( proto = name, visibility = ["//visibility:public"], deps = depset([_go_proto_mapping(dep) for dep in deps] + [ - "@com_github_golang_protobuf//ptypes:go_default_library", - "@com_github_golang_protobuf//ptypes/any:go_default_library", - "@com_github_golang_protobuf//ptypes/duration:go_default_library", - "@com_github_golang_protobuf//ptypes/struct:go_default_library", - "@com_github_golang_protobuf//ptypes/timestamp:go_default_library", - "@com_github_golang_protobuf//ptypes/wrappers:go_default_library", "@com_envoyproxy_protoc_gen_validate//validate:go_default_library", - "@com_google_googleapis//google/api:annotations_go_proto", - "@com_google_googleapis//google/rpc:status_go_proto", + "@com_github_golang_protobuf//ptypes:go_default_library_gen", + "@go_googleapis//google/api:annotations_go_proto", + "@go_googleapis//google/rpc:status_go_proto", + "@io_bazel_rules_go//proto/wkt:any_go_proto", + "@io_bazel_rules_go//proto/wkt:duration_go_proto", + "@io_bazel_rules_go//proto/wkt:struct_go_proto", + "@io_bazel_rules_go//proto/wkt:timestamp_go_proto", + "@io_bazel_rules_go//proto/wkt:wrappers_go_proto", ]).to_list(), ) diff --git a/generated_api_shadow/bazel/repository_locations.bzl b/generated_api_shadow/bazel/repository_locations.bzl index f0173e58946e..68ea3e2ffb84 100644 --- a/generated_api_shadow/bazel/repository_locations.bzl +++ b/generated_api_shadow/bazel/repository_locations.bzl @@ -14,23 +14,30 @@ REPOSITORY_LOCATIONS_SPEC = dict( project_name = "protoc-gen-validate (PGV)", project_desc = "protoc plugin to generate polyglot message validators", project_url = "https://github.com/envoyproxy/protoc-gen-validate", - version = "278964a8052f96a2f514add0298098f63fb7f47f", - sha256 = "e368733c9fb7f8489591ffaf269170d7658cc0cd1ee322b601512b769446d3c8", + version = "1bcea29601b5624234a19b3d7f0ebd9e9984f583", + sha256 = "2062bbe50eddf3c98490339721fb02b5b5cd78f610f163b98bbf95ba7105553f", strip_prefix = "protoc-gen-validate-{version}", urls = ["https://github.com/envoyproxy/protoc-gen-validate/archive/{version}.tar.gz"], - release_date = "2020-06-08", + release_date = "2020-11-30", use_category = ["api"], + implied_untracked_deps = [ + "com_github_iancoleman_strcase", + "com_github_lyft_protoc_gen_star", + "com_github_spf13_afero", + "org_golang_google_genproto", + "org_golang_x_text", + ], ), com_github_cncf_udpa = dict( project_name = "xDS API", project_desc = "xDS API Working Group (xDS-WG)", project_url = "https://github.com/cncf/udpa", # During the UDPA -> xDS migration, we aren't working with releases. - version = "5459f2c994033b0afed7e4a70ac7e90c90c1ffee", - sha256 = "c1f5c2438cf725b5f66aa4210dbc4bb691020c5ed4f64d2bc6638b06a11482f1", + version = "cc1b757b3eddccaaaf0743cbb107742bb7e3ee4f", + sha256 = "822a007cf155855d0c08a2e753a39e222e5816b904436196244066a818a8a230", strip_prefix = "udpa-{version}", urls = ["https://github.com/cncf/udpa/archive/{version}.tar.gz"], - release_date = "2020-11-20", + release_date = "2020-12-11", use_category = ["api"], ), com_github_openzipkin_zipkinapi = dict( diff --git a/source/common/config/protobuf_link_hacks.h b/source/common/config/protobuf_link_hacks.h index b613d60ff84c..de6e816e37b9 100644 --- a/source/common/config/protobuf_link_hacks.h +++ b/source/common/config/protobuf_link_hacks.h @@ -47,6 +47,7 @@ const envoy::service::route::v3::RdsDummy _rds_dummy_v3; const envoy::service::cluster::v3::CdsDummy _cds_dummy_v3; const envoy::service::endpoint::v3::EdsDummy _eds_dummy_v3; const envoy::service::route::v3::SrdsDummy _srds_dummy_v3; +const envoy::service::extension::v3::EcdsDummy _ecds_dummy_v3; // With the v2 -> v3 migration there is another, related linking issue. // Symbols for v2 protos which headers are not included in any file in the codebase are being diff --git a/test/common/access_log/access_log_impl_test.cc b/test/common/access_log/access_log_impl_test.cc index 663f35cd1105..1e6eb05c0cea 100644 --- a/test/common/access_log/access_log_impl_test.cc +++ b/test/common/access_log/access_log_impl_test.cc @@ -1137,7 +1137,7 @@ name: accesslog )EOF"; EXPECT_THROW_WITH_REGEX(AccessLogFactory::fromProto(parseAccessLogFromV3Yaml(yaml), context_), - EnvoyException, ".*\"NOT_A_VALID_CODE\" for type TYPE_ENUM.*"); + EnvoyException, "NOT_A_VALID_CODE"); } TEST_F(AccessLogImplTest, GrpcStatusFilterBlock) { diff --git a/test/common/upstream/cluster_manager_impl_test.cc b/test/common/upstream/cluster_manager_impl_test.cc index d957311d5e34..bd96f50e0c1e 100644 --- a/test/common/upstream/cluster_manager_impl_test.cc +++ b/test/common/upstream/cluster_manager_impl_test.cc @@ -302,7 +302,7 @@ TEST_F(ClusterManagerImplTest, UnknownClusterType) { )EOF"; EXPECT_THROW_WITH_REGEX(create(parseBootstrapFromV3Json(json)), EnvoyException, - "invalid value \"foo\" for type TYPE_ENUM"); + "invalid value \"foo\""); } TEST_F(ClusterManagerImplTest, LocalClusterNotDefined) { diff --git a/test/common/upstream/eds_speed_test.cc b/test/common/upstream/eds_speed_test.cc index d939e3513dd8..33bd77353ad4 100644 --- a/test/common/upstream/eds_speed_test.cc +++ b/test/common/upstream/eds_speed_test.cc @@ -10,6 +10,7 @@ #include "common/config/grpc_mux_impl.h" #include "common/config/grpc_subscription_impl.h" +#include "common/config/protobuf_link_hacks.h" #include "common/config/utility.h" #include "common/singleton/manager_impl.h" #include "common/upstream/eds.h" diff --git a/test/common/upstream/upstream_impl_test.cc b/test/common/upstream/upstream_impl_test.cc index 27d050861af2..7d031bd37506 100644 --- a/test/common/upstream/upstream_impl_test.cc +++ b/test/common/upstream/upstream_impl_test.cc @@ -1917,7 +1917,7 @@ TEST_F(StaticClusterImplTest, UnsupportedLBType) { socket_address: { address: 192.168.1.2, port_value: 44 } )EOF"; - EXPECT_THROW_WITH_MESSAGE( + EXPECT_THROW_WITH_REGEX( { envoy::config::cluster::v3::Cluster cluster_config = parseClusterFromV3Yaml(yaml); Envoy::Stats::ScopePtr scope = @@ -1930,10 +1930,7 @@ TEST_F(StaticClusterImplTest, UnsupportedLBType) { StaticClusterImpl cluster(cluster_config, runtime_, factory_context, std::move(scope), false); }, - EnvoyException, - "Protobuf message (type envoy.config.cluster.v3.Cluster reason " - "INVALID_ARGUMENT:(lb_policy): invalid " - "value \"fakelbtype\" for type TYPE_ENUM) has unknown fields"); + EnvoyException, "invalid value \"fakelbtype\""); } TEST_F(StaticClusterImplTest, MalformedHostIP) { diff --git a/test/extensions/filters/http/compressor/config_test.cc b/test/extensions/filters/http/compressor/config_test.cc index cea48bc00cff..aec2806eb9b3 100644 --- a/test/extensions/filters/http/compressor/config_test.cc +++ b/test/extensions/filters/http/compressor/config_test.cc @@ -1,5 +1,6 @@ #include "extensions/filters/http/compressor/config.h" +#include "test/extensions/filters/http/compressor/mock_compressor_library.pb.h" #include "test/mocks/server/factory_context.h" #include "gtest/gtest.h" @@ -12,6 +13,8 @@ namespace { using testing::NiceMock; +const ::test::mock_compressor_library::Unregistered _mock_compressor_library_dummy; + TEST(CompressorFilterFactoryTests, MissingCompressorLibraryConfig) { const envoy::extensions::filters::http::compressor::v3::Compressor proto_config; CompressorFilterFactory factory; diff --git a/test/integration/BUILD b/test/integration/BUILD index 036286910f1d..0778f624670d 100644 --- a/test/integration/BUILD +++ b/test/integration/BUILD @@ -892,6 +892,7 @@ envoy_cc_test( "//test/mocks/http:http_mocks", "//test/test_common:utility_lib", "@envoy_api//envoy/config/bootstrap/v3:pkg_cc_proto", + "@envoy_api//envoy/config/filter/http/grpc_http1_bridge/v2:pkg_cc_proto", "@envoy_api//envoy/config/route/v3:pkg_cc_proto", "@envoy_api//envoy/extensions/filters/network/http_connection_manager/v3:pkg_cc_proto", ], diff --git a/test/integration/integration_test.cc b/test/integration/integration_test.cc index 15f7587a2d34..deefe009901e 100644 --- a/test/integration/integration_test.cc +++ b/test/integration/integration_test.cc @@ -3,6 +3,7 @@ #include #include "envoy/config/bootstrap/v3/bootstrap.pb.h" +#include "envoy/config/filter/http/grpc_http1_bridge/v2/config.pb.h" #include "envoy/config/route/v3/route_components.pb.h" #include "envoy/extensions/filters/network/http_connection_manager/v3/http_connection_manager.pb.h" @@ -418,6 +419,8 @@ TEST_P(IntegrationTest, UpstreamDisconnectWithTwoRequests) { test_server_->waitForCounterGe("cluster.cluster_0.upstream_rq_200", 2); } +const ::envoy::config::filter::http::grpc_http1_bridge::v2::Config _grpc_http1_bridge_dummy; + // Test hitting the bridge filter with too many response bytes to buffer. Given // the headers are not proxied, the connection manager will send a local error reply. TEST_P(IntegrationTest, HittingGrpcFilterLimitBufferingHeaders) {