Merge pull request #8652 from haberman/sync-stage

Integrate from Piper for C++, Java, and Python
protocolbuffers · Jun 1, 2021 · bd42fcc · bd42fcc
2 parents fd896ba + 45bb1d2
commit bd42fcc
Show file tree

Hide file tree

Showing 49 changed files with 1,650 additions and 292 deletions.
diff --git a/BUILD b/BUILD
@@ -135,6 +135,7 @@ cc_library(
         "src/google/protobuf/arena.cc",
         "src/google/protobuf/arenastring.cc",
         "src/google/protobuf/extension_set.cc",
+        "src/google/protobuf/field_access_listener.cc",
         "src/google/protobuf/generated_enum_util.cc",
         "src/google/protobuf/generated_message_table_driven_lite.cc",
         "src/google/protobuf/generated_message_util.cc",

diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,15 +1,15 @@
 Unreleased Changes (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
-  Protocol Compiler
+  C++
+  * Introduce FieldAccessListener.
   * Stop emitting boilerplate {Copy/Merge}From in each ProtoBuf class
-  * split the accessor annotations according to their operation
-  * introduce proto message injector
-  * let proto message injector decide whether to calculate address info based on field descriptor and access type.
-  * Disable LITE_RUNTIME injector annotations
-  * move callback and @protoc_insertion_point after internal set of enum fields
-  * Improve ExtractFieldInfo codegen for string fields with oneof or default value
-  * Rename MessageInjector to FieldAccessListener
-  * Change the API of FieldAccessListener to support callbacks for info extraction
-  * make field_access_injector private
+  * Fixed some uninitialized variable warnings in generated_message_reflection.cc.
+
+  Kotlin:
+  * Add extension functions related to ByteString to c.g.protobuf.kotlin.
+
+  Java
+  * Fixed parser to check that we are at a proper limit when a sub-message has
+    finished parsing.
 
 2021-05-07 version 3.17.1 (C++/Java/Python/PHP/Objective-C/C#/Ruby/JavaScript)
   PHP

diff --git a/cmake/extract_includes.bat.in b/cmake/extract_includes.bat.in
@@ -46,6 +46,7 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\dynamic_message.h" in
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\empty.pb.h" include\google\protobuf\empty.pb.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set.h" include\google\protobuf\extension_set.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\extension_set_inl.h" include\google\protobuf\extension_set_inl.h
+copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_access_listener.h" include\google\protobuf\field_access_listener.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\field_mask.pb.h" include\google\protobuf\field_mask.pb.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_reflection.h" include\google\protobuf\generated_enum_reflection.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_enum_util.h" include\google\protobuf\generated_enum_util.h

diff --git a/cmake/libprotobuf-lite.cmake b/cmake/libprotobuf-lite.cmake
@@ -3,6 +3,7 @@ set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/arena.cc
   ${protobuf_source_dir}/src/google/protobuf/arenastring.cc
   ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
+  ${protobuf_source_dir}/src/google/protobuf/field_access_listener.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_util.cc

diff --git a/cmake/libprotobuf.cmake b/cmake/libprotobuf.cmake
@@ -65,6 +65,7 @@ set(libprotobuf_includes
   ${protobuf_source_dir}/src/google/protobuf/duration.pb.h
   ${protobuf_source_dir}/src/google/protobuf/dynamic_message.h
   ${protobuf_source_dir}/src/google/protobuf/empty.pb.h
+  ${protobuf_source_dir}/src/google/protobuf/field_access_listener.h
   ${protobuf_source_dir}/src/google/protobuf/field_mask.pb.h
   ${protobuf_source_dir}/src/google/protobuf/generated_message_reflection.h
   ${protobuf_source_dir}/src/google/protobuf/io/gzip_stream.h

diff --git a/conformance/failure_list_java.txt b/conformance/failure_list_java.txt
@@ -42,7 +42,3 @@ Required.Proto3.JsonInput.Int32FieldPlusSign
 Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotBool
 Required.Proto3.JsonInput.RepeatedFieldWrongElementTypeExpectingStringsGotInt
 Required.Proto3.JsonInput.StringFieldNotAString
-Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
-Required.Proto3.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
-Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownNonRepeatedValue.MESSAGE
-Required.Proto2.ProtobufInput.PrematureEofInDelimitedDataForKnownRepeatedValue.MESSAGE
diff --git a/java/core/src/main/java/com/google/protobuf/CodedInputStream.java b/java/core/src/main/java/com/google/protobuf/CodedInputStream.java
@@ -873,6 +873,9 @@ public void readMessage(
       builder.mergeFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
     }
 
@@ -889,6 +892,9 @@ public <T extends MessageLite> T readMessage(
       T result = parser.parsePartialFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
       return result;
     }
@@ -1595,6 +1601,9 @@ public void readMessage(
       builder.mergeFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
     }
 
@@ -1611,6 +1620,9 @@ public <T extends MessageLite> T readMessage(
       T result = parser.parsePartialFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
       return result;
     }
@@ -2392,6 +2404,9 @@ public void readMessage(
       builder.mergeFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
     }
 
@@ -2408,6 +2423,9 @@ public <T extends MessageLite> T readMessage(
       T result = parser.parsePartialFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
       return result;
     }
@@ -3489,6 +3507,9 @@ public void readMessage(
       builder.mergeFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
     }
 
@@ -3505,6 +3526,9 @@ public <T extends MessageLite> T readMessage(
       T result = parser.parsePartialFrom(this, extensionRegistry);
       checkLastTagWas(0);
       --recursionDepth;
+      if (getBytesUntilLimit() != 0) {
+        throw InvalidProtocolBufferException.truncatedMessage();
+      }
       popLimit(oldLimit);
       return result;
     }

diff --git a/python/google/protobuf/pyext/message.cc b/python/google/protobuf/pyext/message.cc
@@ -1564,12 +1564,17 @@ static int InternalReparentFields(
                                            to_release);
   }
 
-  GOOGLE_CHECK_EQ(self->message->GetArena(), new_message->message->GetArena());
-
-  MessageReflectionFriend::UnsafeShallowSwapFields(
-      self->message, new_message->message,
-      std::vector<const FieldDescriptor*>(fields_to_swap.begin(),
-                                          fields_to_swap.end()));
+  if (self->message->GetArena() == new_message->message->GetArena()) {
+    MessageReflectionFriend::UnsafeShallowSwapFields(
+        self->message, new_message->message,
+        std::vector<const FieldDescriptor*>(fields_to_swap.begin(),
+                                            fields_to_swap.end()));
+  } else {
+    self->message->GetReflection()->SwapFields(
+        self->message, new_message->message,
+        std::vector<const FieldDescriptor*>(fields_to_swap.begin(),
+                                            fields_to_swap.end()));
+  }
 
   // This might delete the Python message completely if all children were moved.
   Py_DECREF(self);

diff --git a/src/Makefile.am b/src/Makefile.am
@@ -95,6 +95,7 @@ nobase_include_HEADERS =                                         \
   google/protobuf/empty.pb.h                                     \
   google/protobuf/extension_set.h                                \
   google/protobuf/extension_set_inl.h                            \
+  google/protobuf/field_access_listener.h                        \
   google/protobuf/field_mask.pb.h                                \
   google/protobuf/generated_enum_reflection.h                    \
   google/protobuf/generated_enum_util.h                          \
@@ -206,6 +207,7 @@ libprotobuf_lite_la_SOURCES =                                  \
   google/protobuf/arena.cc                                     \
   google/protobuf/arenastring.cc                               \
   google/protobuf/extension_set.cc                             \
+  google/protobuf/field_access_listener.cc                     \
   google/protobuf/generated_enum_util.cc                       \
   google/protobuf/generated_message_util.cc                    \
   google/protobuf/generated_message_table_driven_lite.h        \

diff --git a/src/google/protobuf/any.pb.cc b/src/google/protobuf/any.pb.cc