Merge pull request #7925 from haberman/sync-stage

Integrate from Piper for C++, Java, and Python

BUILD

@@ -164,6 +164,7 @@ cc_library(
         # AUTOGEN(protobuf_lite_srcs)
         "src/google/protobuf/any_lite.cc",
         "src/google/protobuf/arena.cc",
+        "src/google/protobuf/arenastring.cc",
         "src/google/protobuf/extension_set.cc",
         "src/google/protobuf/generated_enum_util.cc",
         "src/google/protobuf/generated_message_table_driven_lite.cc",

CHANGES.txt

@@ -3,10 +3,13 @@ Unreleased Changes
   Protocol Compiler
   * The proto compiler no longer requires a .proto filename when it is not
     generating code.
+  * Added flag `--deterministic_output` to `protoc --encode=...`.
+  * Fixed deadlock when using google.protobuf.Any embedded in aggregate options.
 
   C++
   * Arenas are now unconditionally enabled. cc_enable_arenas no longer has
     any effect.
+  * Removed inlined string support, which is incompatible with arenas.
   * Fix a memory corruption bug in reflection when mixing optional and
     non-optional fields.
   * Make SpaceUsed() calculation more thorough for map fields.
@@ -20,9 +23,23 @@ Unreleased Changes
   * When running under ASAN, skip a test that makes huge allocations.
   * Fixed a crash that could happen when creating more than 256 extensions in
     a single message.
+  * Fix a crash in BuildFile when passing in invalid descriptor proto.
+  * Parser security fix when operating with CodedInputStream.
+  * Warn against the use of AllowUnknownExtension.
+  * Migrated to C++11 for-range loops instead of index-based loops where
+    possible. This fixes a lot of warnings when compiling with -Wsign-compare.
 
   Java
   * Bugfix in mergeFrom() when a oneof has multiple message fields.
+  * Fix RopeByteString.RopeInputStream.read() returning -1 when told to read
+    0 bytes when not at EOF.
+  * Redefine remove(Object) on primitive repeated field Lists to avoid
+    autoboxing.
+  * Support "\u" escapes in textformat string literals.
+  * Trailing empty spaces are no longer ignored for FieldMask.
+  * Fix FieldMaskUtil.subtract to recursively remove mask.
+  * Mark enums with `@java.lang.Deprecated` if the proto enum has option
+    `deprecated = true;`.
 
   Python
   * Print google.protobuf.NullValue as null instead of "NULL_VALUE" when it is

cmake/extract_includes.bat.in

@@ -53,7 +53,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_tab
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\generated_message_util.h" include\google\protobuf\generated_message_util.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\has_bits.h" include\google\protobuf\has_bits.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\implicit_weak_message.h" include\google\protobuf\implicit_weak_message.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\inlined_string_field.h" include\google\protobuf\inlined_string_field.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\coded_stream.h" include\google\protobuf\io\coded_stream.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\gzip_stream.h" include\google\protobuf\io\gzip_stream.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\io\io_win32.h" include\google\protobuf\io\io_win32.h
@@ -88,7 +87,6 @@ copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\bytestream.h" i
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\callback.h" include\google\protobuf\stubs\callback.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\casts.h" include\google\protobuf\stubs\casts.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\common.h" include\google\protobuf\stubs\common.h
-copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\fastmem.h" include\google\protobuf\stubs\fastmem.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\hash.h" include\google\protobuf\stubs\hash.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\logging.h" include\google\protobuf\stubs\logging.h
 copy "${PROTOBUF_SOURCE_WIN32_PATH}\..\src\google\protobuf\stubs\macros.h" include\google\protobuf\stubs\macros.h

cmake/libprotobuf-lite.cmake

@@ -1,6 +1,7 @@
 set(libprotobuf_lite_files
   ${protobuf_source_dir}/src/google/protobuf/any_lite.cc
   ${protobuf_source_dir}/src/google/protobuf/arena.cc
+  ${protobuf_source_dir}/src/google/protobuf/arenastring.cc
   ${protobuf_source_dir}/src/google/protobuf/extension_set.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_enum_util.cc
   ${protobuf_source_dir}/src/google/protobuf/generated_message_table_driven_lite.cc

conformance/Makefile.am

@@ -333,7 +333,7 @@ conformance-php-c:
 
 # Targets for actually running tests.
 test_cpp: protoc_middleman conformance-test-runner conformance-cpp
-	./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt ./conformance-cpp
+	./conformance-test-runner --enforce_recommended --failure_list failure_list_cpp.txt --text_format_failure_list text_format_failure_list_cpp.txt ./conformance-cpp
 
 test_java: protoc_middleman conformance-test-runner conformance-java
 	./conformance-test-runner --enforce_recommended --failure_list failure_list_java.txt --text_format_failure_list text_format_failure_list_java.txt ./conformance-java
@@ -359,10 +359,24 @@ test_php_c_32: protoc_middleman conformance-test-runner conformance-php-c $(othe
 # These depend on library paths being properly set up.  The easiest way to
 # run them is to just use "tox" from the python dir.
 test_python: protoc_middleman conformance-test-runner
-	./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py
+	VERSION="$(shell python --version 2>&1)"; \
+	if [[ "$$VERSION"  == "Python 2.7"* ]]; then \
+	  echo "Using Python 2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python_2.7.txt ./conformance_python.py; \
+	else \
+	  echo "Using Python >2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python.txt --text_format_failure_list text_format_failure_list_python.txt ./conformance_python.py; \
+	fi
 
 test_python_cpp: protoc_middleman conformance-test-runner
-	./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt ./conformance_python.py
+	VERSION="$(shell python --version 2>&1)"; \
+	if [[ "$$VERSION"  == "Python 2.7"* ]]; then \
+	  echo "Using Python 2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp_2.7.txt ./conformance_python.py; \
+	else \
+	  echo "Using Python >2.7 failure list."; \
+	  ./conformance-test-runner --enforce_recommended --failure_list failure_list_python_cpp.txt --text_format_failure_list text_format_failure_list_python_cpp.txt ./conformance_python.py; \
+	fi
 
 test_nodejs: protoc_middleman conformance-test-runner $(other_language_protoc_outputs)
 	NODE_PATH=../js:. ./conformance-test-runner --enforce_recommended --failure_list failure_list_js.txt ./conformance_nodejs.js

conformance/failure_list_js.txt

@@ -1 +0,0 @@
-

conformance/text_format_conformance_suite.cc

@@ -258,6 +258,84 @@ void TextFormatConformanceTestSuite::RunSuiteImpl() {
   RunValidTextFormatTest("FloatFieldLargerThanUint64", REQUIRED,
                          "optional_float: 18446744073709551616");
 
+  // String literals x {Strings, Bytes}
+  for (const auto& field_type : std::vector<std::string>{"String", "Bytes"}) {
+    const std::string field_name =
+        field_type == "String" ? "optional_string" : "optional_bytes";
+    RunValidTextFormatTest(
+        StrCat("StringLiteralConcat", field_type), REQUIRED,
+        StrCat(field_name, ": 'first' \"second\"\n'third'"));
+    RunValidTextFormatTest(
+        StrCat("StringLiteralBasicEscapes", field_type), REQUIRED,
+        StrCat(field_name, ": '\\a\\b\\f\\n\\r\\t\\v\\?\\\\\\'\\\"'"));
+    RunValidTextFormatTest(
+        StrCat("StringLiteralOctalEscapes", field_type), REQUIRED,
+        StrCat(field_name, ": '\\341\\210\\264'"));
+    RunValidTextFormatTest(StrCat("StringLiteralHexEscapes", field_type),
+                           REQUIRED,
+                           StrCat(field_name, ": '\\xe1\\x88\\xb4'"));
+    RunValidTextFormatTest(
+        StrCat("StringLiteralShortUnicodeEscape", field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\u1234'"));
+    RunValidTextFormatTest(
+        StrCat("StringLiteralLongUnicodeEscapes", field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\U00001234\\U00010437'"));
+    // String literals don't include line feeds.
+    ExpectParseFailure(StrCat("StringLiteralIncludesLF", field_type),
+                       REQUIRED,
+                       StrCat(field_name, ": 'first line\nsecond line'"));
+    // Unicode escapes don't include code points that lie beyond the planes
+    // (> 0x10ffff).
+    ExpectParseFailure(
+        StrCat("StringLiteralLongUnicodeEscapeTooLarge", field_type),
+        REQUIRED, StrCat(field_name, ": '\\U00110000'"));
+    // Unicode escapes don't include surrogates.
+    ExpectParseFailure(
+        StrCat("StringLiteralShortUnicodeEscapeSurrogatePair",
+                     field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\ud801\\udc37'"));
+    ExpectParseFailure(
+        StrCat("StringLiteralShortUnicodeEscapeSurrogateFirstOnly",
+                     field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\ud800'"));
+    ExpectParseFailure(
+        StrCat("StringLiteralShortUnicodeEscapeSurrogateSecondOnly",
+                     field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\udc00'"));
+    ExpectParseFailure(
+        StrCat("StringLiteralLongUnicodeEscapeSurrogateFirstOnly",
+                     field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\U0000d800'"));
+    ExpectParseFailure(
+        StrCat("StringLiteralLongUnicodeEscapeSurrogateSecondOnly",
+                     field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\U0000dc00'"));
+    ExpectParseFailure(
+        StrCat("StringLiteralLongUnicodeEscapeSurrogatePair", field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\U0000d801\\U00000dc37'"));
+    ExpectParseFailure(
+        StrCat("StringLiteralUnicodeEscapeSurrogatePairLongShort",
+                     field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\U0000d801\\udc37'"));
+    ExpectParseFailure(
+        StrCat("StringLiteralUnicodeEscapeSurrogatePairShortLong",
+                     field_type),
+        RECOMMENDED, StrCat(field_name, ": '\\ud801\\U0000dc37'"));
+
+    // The following method depend on the type of field, as strings have extra
+    // validation.
+    const auto test_method =
+        field_type == "String"
+            ? &TextFormatConformanceTestSuite::ExpectParseFailure
+            : &TextFormatConformanceTestSuite::RunValidTextFormatTest;
+
+    // String fields reject invalid UTF-8 byte sequences; bytes fields don't.
+    (this->*test_method)(StrCat(field_type, "FieldBadUTF8Octal"),
+                         REQUIRED, StrCat(field_name, ": '\\300'"));
+    (this->*test_method)(StrCat(field_type, "FieldBadUTF8Hex"), REQUIRED,
+                         StrCat(field_name, ": '\\xc0'"));
+  }
+
   // Group fields
   RunValidTextFormatTestProto2("GroupFieldNoColon", REQUIRED,
                                "Data { group_int32: 1 }");

conformance/text_format_failure_list_cpp.txt

@@ -0,0 +1,20 @@
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString

conformance/text_format_failure_list_java.txt

@@ -4,3 +4,10 @@ Recommended.Proto3.ProtobufInput.RepeatedUnknownFields_Drop.TextFormatOutput
 Recommended.Proto3.ProtobufInput.ScalarUnknownFields_Drop.TextFormatOutput
 Required.Proto3.TextFormatInput.AnyField.ProtobufOutput
 Required.Proto3.TextFormatInput.AnyField.TextFormatOutput
+
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Hex
+Required.Proto3.TextFormatInput.StringFieldBadUTF8Octal
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

conformance/text_format_failure_list_python.txt

@@ -3,3 +3,32 @@
 # TODO: These should be fixed.
 Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
 Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
+
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput

conformance/text_format_failure_list_python_2.7.txt

@@ -0,0 +1,36 @@
+# This is the list of text format conformance tests that are known to fail right
+# now.
+# TODO: These should be fixed.
+Required.Proto3.TextFormatInput.FloatFieldMaxValue.ProtobufOutput
+Required.Proto3.TextFormatInput.FloatFieldMaxValue.TextFormatOutput
+
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapesString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeBytes.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.ProtobufOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeString.TextFormatOutput
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateFirstOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogatePairString
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyBytes
+Recommended.Proto3.TextFormatInput.StringLiteralShortUnicodeEscapeSurrogateSecondOnlyString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairLongShortString
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongBytes
+Recommended.Proto3.TextFormatInput.StringLiteralUnicodeEscapeSurrogatePairShortLongString
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesBytes.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.ProtobufOutput
+Required.Proto3.TextFormatInput.StringLiteralBasicEscapesString.TextFormatOutput
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeBytes
+Required.Proto3.TextFormatInput.StringLiteralLongUnicodeEscapeTooLargeString