Merge branch 'kotlinProtos' of https://github.com/deannagarcia/protobuf…

… into kotlinProtos

.github/workflows/codespell.yml

@@ -13,4 +13,4 @@ jobs:
         with:
           check_filenames: true
           skip: ./.git,./conformance/third_party,*.snk,*.pb,*.pb.cc,*.pb.h,./src/google/protobuf/testdata,./objectivec/Tests,./python/compatibility_tests/v2.5.0/tests/google/protobuf/internal
-          ignore_words_list: "alow,alse,ba,cleare,copyable,cloneable,dedup,dur,errorprone,files',fo,fundementals,hel,importd,inout,leapyear,nd,nin,ois,ons,parseable,process',te,testof,ue,unparseable,wasn,wee,gae,keyserver,objext,od,optin"
+          ignore_words_list: "alow,alse,ba,cleare,copyable,cloneable,dedup,dur,errorprone,files',fo,fundementals,hel,importd,inout,leapyear,nd,nin,ois,ons,parseable,process',te,testof,ue,unparseable,wasn,wee,gae,keyserver,objext,od,optin"

CHANGES.txt

@@ -22,8 +22,11 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
     on an error path.
   * Avoid expensive inlined code space for encoding message length for messages
     >= 128 bytes and instead do a procedure call to a shared out-of-line routine.
+  * util::DefaultFieldComparator will be final in a future version of protobuf.
+    Subclasses should inherit from SimpleFieldComparator instead.
 
   Java:
+  * Detect invalid overflow of byteLimit and return InvalidProtocolBufferException as documented.
   * Exceptions thrown while reading from an InputStream in parseFrom are now
     included as causes.
   * Support potentially more efficient proto parsing from RopeByteStrings.
@@ -37,6 +40,15 @@ Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
   JavaScript
   * Make Any.pack() chainable.
 
+2021-04-02 version 3.15.7 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
+
+  C++
+  * Remove the ::pb namespace (alias) (#8423)
+
+  Ruby
+  * Fix unbounded memory growth for Ruby <2.7 (#8429)
+  * Fixed message equality in cases where the message type is different (#8434)
+
 2021-03-10 version 3.15.6 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
 
   Ruby

Protobuf-C++.podspec

@@ -1,6 +1,6 @@
 Pod::Spec.new do |s|
   s.name     = 'Protobuf-C++'
-  s.version  = '3.15.6'
+  s.version  = '3.15.7'
   s.summary  = 'Protocol Buffers v3 runtime library for C++.'
   s.homepage = 'https://github.com/google/protobuf'
   s.license  = '3-Clause BSD License'

Protobuf.podspec

@@ -5,7 +5,7 @@
 # dependent projects use the :git notation to refer to the library.
 Pod::Spec.new do |s|
   s.name     = 'Protobuf'
-  s.version  = '3.15.6'
+  s.version  = '3.15.7'
   s.summary  = 'Protocol Buffers v.3 runtime library for Objective-C.'
   s.homepage = 'https://github.com/protocolbuffers/protobuf'
   s.license  = '3-Clause BSD License'

configure.ac

@@ -17,7 +17,7 @@ AC_PREREQ(2.59)
 # In the SVN trunk, the version should always be the next anticipated release
 # version with the "-pre" suffix.  (We used to use "-SNAPSHOT" but this pushed
 # the size of one file name in the dist tarfile over the 99-char limit.)
-AC_INIT([Protocol Buffers],[3.15.6],[protobuf@googlegroups.com],[protobuf])
+AC_INIT([Protocol Buffers],[3.15.7],[protobuf@googlegroups.com],[protobuf])
 
 AM_MAINTAINER_MODE([enable])
 

csharp/Google.Protobuf.Tools.nuspec

@@ -5,7 +5,7 @@
     <title>Google Protocol Buffers tools</title>
     <summary>Tools for Protocol Buffers - Google's data interchange format.</summary>
     <description>See project site for more info.</description>
-    <version>3.15.6</version>
+    <version>3.15.7</version>
     <authors>Google Inc.</authors>
     <owners>protobuf-packages</owners>
     <licenseUrl>https://github.com/protocolbuffers/protobuf/blob/master/LICENSE</licenseUrl>

csharp/src/Google.Protobuf/Google.Protobuf.csproj

@@ -4,7 +4,7 @@
     <Description>C# runtime library for Protocol Buffers - Google's data interchange format.</Description>
     <Copyright>Copyright 2015, Google Inc.</Copyright>
     <AssemblyTitle>Google Protocol Buffers</AssemblyTitle>
-    <VersionPrefix>3.15.6</VersionPrefix>
+    <VersionPrefix>3.15.7</VersionPrefix>
     <!-- C# 7.2 is required for Span/BufferWriter/ReadOnlySequence -->
     <LangVersion>7.2</LangVersion>
     <Authors>Google Inc.</Authors>

csharp/src/Google.Protobuf/Reflection/Descriptor.cs

@@ -4796,11 +4796,11 @@ public sealed partial class FileOptions : pb::IExtendableMessage<FileOptions>
 
     private string javaOuterClassname_;
     /// <summary>
-    /// If set, all the classes from the .proto file are wrapped in a single
-    /// outer class with the given name.  This applies to both Proto1
-    /// (equivalent to the old "--one_java_file" option) and Proto2 (where
-    /// a .proto always translates to a single class, but you may want to
-    /// explicitly choose the class name).
+    /// Controls the name of the wrapper Java class generated for the .proto file.
+    /// That class will always contain the .proto file's getDescriptor() method as
+    /// well as any top-level extensions defined in the .proto file.
+    /// If java_multiple_files is disabled, then all the other classes from the
+    /// .proto file will be nested inside the single wrapper outer class.
     /// </summary>
     [global::System.Diagnostics.DebuggerNonUserCodeAttribute]
     public string JavaOuterClassname {
@@ -4826,10 +4826,10 @@ public sealed partial class FileOptions : pb::IExtendableMessage<FileOptions>
 
     private bool javaMultipleFiles_;
     /// <summary>
-    /// If set true, then the Java code generator will generate a separate .java
+    /// If enabled, then the Java code generator will generate a separate .java
     /// file for each top-level message, enum, and service defined in the .proto
-    /// file.  Thus, these types will *not* be nested inside the outer class
-    /// named by java_outer_classname.  However, the outer class will still be
+    /// file.  Thus, these types will *not* be nested inside the wrapper class
+    /// named by java_outer_classname.  However, the wrapper class will still be
     /// generated to contain the file's getDescriptor() method as well as any
     /// top-level extensions defined in the file.
     /// </summary>

examples/AddPerson.java

@@ -1,7 +1,7 @@
 // See README.txt for information and build instructions.
 
-import com.example.tutorial.AddressBookProtos.AddressBook;
-import com.example.tutorial.AddressBookProtos.Person;
+import com.example.tutorial.protos.AddressBook;
+import com.example.tutorial.protos.Person;
 import java.io.BufferedReader;
 import java.io.FileInputStream;
 import java.io.FileNotFoundException;

examples/ListPeople.java

@@ -1,7 +1,7 @@
 // See README.txt for information and build instructions.
 
-import com.example.tutorial.AddressBookProtos.AddressBook;
-import com.example.tutorial.AddressBookProtos.Person;
+import com.example.tutorial.protos.AddressBook;
+import com.example.tutorial.protos.Person;
 import java.io.FileInputStream;
 import java.io.IOException;
 import java.io.PrintStream;

examples/addressbook.proto

@@ -14,7 +14,8 @@ import "google/protobuf/timestamp.proto";
 // [END declaration]
 
 // [START java_declaration]
-option java_package = "com.example.tutorial";
+option java_multiple_files = true;
+option java_package = "com.example.tutorial.protos";
 option java_outer_classname = "AddressBookProtos";
 // [END java_declaration]
 

java/bom/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-bom</artifactId>
-  <version>3.15.6</version>
+  <version>3.15.7</version>
   <packaging>pom</packaging>
 
   <name>Protocol Buffers [BOM]</name>

java/core/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.15.6</version>
+    <version>3.15.7</version>
   </parent>
 
   <artifactId>protobuf-java</artifactId>

java/core/src/main/java/com/google/protobuf/CodedInputStream.java

@@ -1185,6 +1185,9 @@ public int pushLimit(int byteLimit) throws InvalidProtocolBufferException {
         throw InvalidProtocolBufferException.negativeSize();
       }
       byteLimit += getTotalBytesRead();
+      if (byteLimit < 0) {
+        throw InvalidProtocolBufferException.parseFailure();
+      }
       final int oldLimit = currentLimit;
       if (byteLimit > oldLimit) {
         throw InvalidProtocolBufferException.truncatedMessage();

java/core/src/test/java/com/google/protobuf/CodedInputStreamTest.java

@@ -1284,4 +1284,33 @@ public synchronized int read(byte[] b, int off, int len) {
     maliciousCapture.get(1)[0] = 0x9;
     assertEquals(0x9, byteArray[0]); // MODIFICATION! Should we fix?
   }
+
+  public void testInvalidInputYieldsInvalidProtocolBufferException_readTag() throws Exception {
+    byte[] input = new byte[] {0x0a, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, 0x77};
+    CodedInputStream inputStream = CodedInputStream.newInstance(input);
+    try {
+      inputStream.readTag();
+      int size = inputStream.readRawVarint32();
+      inputStream.pushLimit(size);
+      inputStream.readTag();
+      fail();
+    } catch (InvalidProtocolBufferException ex) {
+      // Expected.
+    }
+  }
+
+  public void testInvalidInputYieldsInvalidProtocolBufferException_readBytes() throws Exception {
+    byte[] input =
+        new byte[] {0x0a, (byte) 0xff, (byte) 0xff, (byte) 0xff, (byte) 0xff, 0x67, 0x1a, 0x1a};
+    CodedInputStream inputStream = CodedInputStream.newInstance(input);
+    try {
+      inputStream.readTag();
+      int size = inputStream.readRawVarint32();
+      inputStream.pushLimit(size);
+      inputStream.readBytes();
+      fail();
+    } catch (InvalidProtocolBufferException ex) {
+      // Expected.
+    }
+  }
 }

java/lite/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.15.6</version>
+    <version>3.15.7</version>
   </parent>
 
   <artifactId>protobuf-javalite</artifactId>

java/pom.xml

@@ -4,7 +4,7 @@
 
   <groupId>com.google.protobuf</groupId>
   <artifactId>protobuf-parent</artifactId>
-  <version>3.15.6</version>
+  <version>3.15.7</version>
   <packaging>pom</packaging>
 
   <name>Protocol Buffers [Parent]</name>

java/util/pom.xml

@@ -4,7 +4,7 @@
   <parent>
     <groupId>com.google.protobuf</groupId>
     <artifactId>protobuf-parent</artifactId>
-    <version>3.15.6</version>
+    <version>3.15.7</version>
   </parent>
 
   <artifactId>protobuf-java-util</artifactId>

js/package.json

@@ -1,6 +1,6 @@
 {
   "name": "google-protobuf",
-  "version": "3.15.6",
+  "version": "3.15.7",
   "description": "Protocol Buffers for JavaScript",
   "main": "google-protobuf.js",
   "files": [

kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../../..
+
+# running dockcross image without any arguments generates a wrapper
+# scripts that can be used to run commands under the dockcross image
+# easily.
+# See https://github.com/dockcross/dockcross#usage for details
+docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh
+chmod +x dockcross-manylinux2014-aarch64.sh
+
+# the wrapper script has CRLF line endings and bash doesn't like that
+# so we change CRLF line endings into LF.
+sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh
+
+# The dockcross wrapper script runs arbitrary commands under the selected dockcross
+# image with the following properties which make its use very convenient:
+# * the current working directory is mounted under /work so the container can easily
+#   access the current workspace
+# * the processes in the container run under the same UID and GID as the host process so unlike
+#   vanilla "docker run" invocations, the workspace doesn't get polluted with files
+#   owned by root.
+./dockcross-manylinux2014-aarch64.sh "$@"

kokoro/linux/aarch64/python_crosscompile_aarch64.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator.
+# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image.
+
+set -ex
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+
+./autogen.sh
+CXXFLAGS="-fPIC -g -O2" ./configure
+make -j8
+
+pushd python
+
+# TODO: currently this step relies on qemu being registered with binfmt_misc so that
+# aarch64 binaries are automatically run with an emulator. This works well once
+# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine.
+${PYTHON} setup.py build_py
+
+# when crosscompiling for aarch64, --plat-name needs to be set explicitly
+# to end up with correctly named wheel file
+# the value should be manylinuxABC_ARCH and dockcross docker image
+# conveniently provides the value in the AUDITWHEEL_PLAT env
+plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
+
+# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
+export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
+
+# Build the python extension inplace to be able to python unittests later
+${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace
+
+# Build the binary wheel (to check it with auditwheel)
+${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag

kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+cd python
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+${PYTHON} -m pip install --user six pytest auditwheel
+
+# check that we are really using aarch64 python
+(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1)
+
+# step 1: run all python unittests
+# we've built the python extension previously with --inplace option
+# so we can just discover all the unittests and run them directly under 
+# the python/ directory.
+LD_LIBRARY_PATH=../src/.libs PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf
+
+# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible.
+# auditwheel needs to run on wheel's target platform (or under an emulator)
+${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl
+
+# step 3: smoketest that the wheel can be installed and run a smokecheck
+${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl
+# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfully loaded the python cpp extension!")'

kokoro/linux/aarch64/test_python_aarch64.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh
+
+# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution
+#   (manylinux_2_24 is debian-based while manylinux2014 is centos-based)
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

php/ext/google/protobuf/package.xml

@@ -10,19 +10,19 @@
   <email>protobuf-opensource@google.com</email>
   <active>yes</active>
  </lead>
- <date>2021-03-10</date>
- <time>10:11:34</time>
+ <date>2021-04-02</date>
+ <time>10:01:42</time>
  <version>
-  <release>3.15.6</release>
-  <api>3.15.6</api>
+  <release>3.15.7</release>
+  <api>3.15.7</api>
  </version>
  <stability>
   <release>stable</release>
   <api>stable</api>
  </stability>
  <license uri="https://opensource.org/licenses/BSD-3-Clause">3-Clause BSD License</license>
  <notes>
- No new changes in 3.15.6
+ No new changes in 3.15.7
  </notes>
  <contents>
   <dir baseinstalldir="/" name="/">
@@ -887,5 +887,20 @@ G  A release.
    <notes>
    </notes>
   </release>
+  <release>
+   <version>
+    <release>3.15.7</release>
+    <api>3.15.7</api>
+   </version>
+   <stability>
+    <release>stable</release>
+    <api>stable</api>
+   </stability>
+   <date>2021-04-02</date>
+   <time>10:01:42</time>
+   <license uri="https://opensource.org/licenses/BSD-3-Clause">3-Clause BSD License</license>
+   <notes>
+   </notes>
+  </release>
  </changelog>
 </package>